Add KJV translator footnotes package

scripts/generate-docs.js

@@ -129,6 +129,7 @@ function resourceDetailMarkdown(manifest, packageCatalog) {
     `- Package catalog: ${manifest.packages?.json_catalog ?? 'n/a'}`,
     `- Verses JSONL: ${manifest.packages?.verses_jsonl ?? 'n/a'}`,
     `- Strong\'s links JSONL: ${manifest.packages?.strongs_jsonl ?? 'n/a'}`,
+    `- Footnotes JSONL: ${manifest.packages?.footnotes_jsonl ?? 'n/a'}`,
     `- Entries JSONL: ${manifest.packages?.entries_jsonl ?? 'n/a'}`,
     `- Source SHA-256: ${packageCatalog?.source_sha256 ?? manifest.checks?.expected_sha256 ?? 'n/a'}`,
     `- Last checked: ${manifest.checks?.last_checked_at ?? 'n/a'}`,
@@ -138,6 +139,7 @@ function resourceDetailMarkdown(manifest, packageCatalog) {
     `- Books: ${packageCatalog?.counts?.books ?? 'n/a'}`,
     `- Verses: ${packageCatalog?.counts?.verses ?? 'n/a'}`,
     `- Strong\'s links: ${packageCatalog?.counts?.strongs_links ?? 'n/a'}`,
+    `- Footnotes: ${packageCatalog?.counts?.footnotes ?? 'n/a'}`,
     `- Entries: ${packageCatalog?.counts?.entries ?? 'n/a'}`,
     `- Commentary entries: ${packageCatalog?.counts?.commentary_entries ?? 'n/a'}`,
     `- Dictionary entries: ${packageCatalog?.counts?.dictionary_entries ?? 'n/a'}`,
@@ -198,6 +200,7 @@ function countsLabel(packageCatalog) {
     counts.books != null ? `${counts.books} books` : null,
     counts.verses != null ? `${counts.verses} verses` : null,
     counts.strongs_links != null ? `${counts.strongs_links} Strong's links` : null,
+    counts.footnotes != null ? `${counts.footnotes} footnotes` : null,
     counts.commentary_entries != null ? `${counts.commentary_entries} commentary entries` : null,
     counts.dictionary_entries != null ? `${counts.dictionary_entries} dictionary entries` : null,
     counts.entries != null && counts.commentary_entries == null && counts.dictionary_entries == null ? `${counts.entries} entries` : null,

scripts/import-usfm.js

@@ -43,7 +43,9 @@ const usfmEntries = zip
 
 const verses = []
 const strongsLinks = []
+const footnotes = []
 let tokenId = 1
+let footnoteId = 1
 
 for (const entry of usfmEntries) {
   const filePath = path.join(extractDir, entry.entryName)
@@ -68,6 +70,7 @@ for (const entry of usfmEntries) {
     const raw = verseMatch[2]
     const verseId = `${bookId}.${chapter}.${verseNumber}`
     const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
+    const notes = extractFootnotes(raw)
     const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))
 
     verses.push({
@@ -95,6 +98,21 @@ for (const entry of usfmEntries) {
         strong: normalizeStrong(link[2]),
       })
     }
+
+    for (const [index, note] of notes.entries()) {
+      footnotes.push({
+        id: footnoteId++,
+        verse_id: verseId,
+        translation_id: manifest.id,
+        book_id: bookId,
+        chapter,
+        verse: verseNumber,
+        note_index: index + 1,
+        caller: note.caller,
+        reference: note.reference || `${chapter}.${verseNumber}`,
+        text: note.text,
+      })
+    }
   }
 }
 
@@ -109,9 +127,15 @@ await writeFile(
   `${strongsLinks.map((link) => JSON.stringify(link)).join('\n')}\n`,
   'utf8',
 )
+await writeFile(
+  path.join(outputDir, 'footnotes.jsonl'),
+  `${footnotes.map((note) => JSON.stringify(note)).join('\n')}\n`,
+  'utf8',
+)
 
 const versesPackage = await packageFile(outputDir, 'verses.jsonl')
 const strongsPackage = await packageFile(outputDir, 'strongs-links.jsonl')
+const footnotesPackage = await packageFile(outputDir, 'footnotes.jsonl')
 
 const catalog = {
   schema_version: 'librebible.resource-catalog.v1',
@@ -140,10 +164,12 @@ const catalog = {
     books: new Set(verses.map((verse) => verse.book_id)).size,
     verses: verses.length,
     strongs_links: strongsLinks.length,
+    footnotes: footnotes.length,
   },
   files: {
     verses_jsonl: versesPackage,
     strongs_jsonl: strongsPackage,
+    footnotes_jsonl: footnotesPackage,
   },
 }
 
@@ -182,6 +208,25 @@ await writeJson(path.join(root, 'packages', 'json', 'catalog.json'), {
 
 console.log(`${manifest.id}: imported ${catalog.counts.verses} verses`)
 console.log(`${manifest.id}: imported ${catalog.counts.strongs_links} Strong's links`)
+console.log(`${manifest.id}: imported ${catalog.counts.footnotes} footnotes`)
+
+function extractFootnotes(raw) {
+  return [...raw.matchAll(/\\f\s+([\s\S]*?)\\f\*/g)]
+    .map((match) => {
+      const content = match[1]
+      const caller = /^([+\w-]+)/.exec(content.trim())?.[1] ?? '+'
+      const reference = /\\fr\s+([^\\]+)/.exec(content)?.[1]?.trim() ?? ''
+      const textSource = content
+        .replace(/^([+\w-]+)/, '')
+        .replace(/\\fr\s+[^\\]+/, '')
+      return {
+        caller,
+        reference,
+        text: cleanUsfmText(textSource),
+      }
+    })
+    .filter((note) => note.text.length > 0)
+}
 
 async function packageFile(outputDir, fileName) {
   const filePath = path.join(outputDir, fileName)

scripts/lib.js

@@ -59,6 +59,11 @@ export function normalizeStrong(value) {
 
 export function cleanUsfmText(input) {
   return input
+    .replace(/…/g, '...')
+    .replace(/…/g, '...')
+    .replace(/’/g, "'")
+    .replace(/“/g, '"')
+    .replace(/”/g, '"')
     .replace(/\\f [\s\S]*?\\f\*/g, '')
     .replace(/\\x [\s\S]*?\\x\*/g, '')
     .replace(/\\add\s+([^\\]+)\\add\*/g, '$1')