Add NET Bible package

2026-07-12 11:11:47 -05:00
parent 84ca97e0cd
commit aad3ca19e9
24 changed files with 691237 additions and 7024 deletions
@@ -130,6 +130,7 @@ function resourceDetailMarkdown(manifest, packageCatalog) {
    `- Verses JSONL: ${manifest.packages?.verses_jsonl ?? 'n/a'}`,
    `- Strong\'s links JSONL: ${manifest.packages?.strongs_jsonl ?? 'n/a'}`,
    `- Footnotes JSONL: ${manifest.packages?.footnotes_jsonl ?? 'n/a'}`,
+    `- Study notes JSONL: ${manifest.packages?.study_notes_jsonl ?? 'n/a'}`,
    `- Entries JSONL: ${manifest.packages?.entries_jsonl ?? 'n/a'}`,
    `- Source SHA-256: ${packageCatalog?.source_sha256 ?? manifest.checks?.expected_sha256 ?? 'n/a'}`,
    `- Last checked: ${manifest.checks?.last_checked_at ?? 'n/a'}`,
@@ -140,6 +141,7 @@ function resourceDetailMarkdown(manifest, packageCatalog) {
    `- Verses: ${packageCatalog?.counts?.verses ?? 'n/a'}`,
    `- Strong\'s links: ${packageCatalog?.counts?.strongs_links ?? 'n/a'}`,
    `- Footnotes: ${packageCatalog?.counts?.footnotes ?? 'n/a'}`,
+    `- Study notes: ${packageCatalog?.counts?.study_notes ?? 'n/a'}`,
    `- Entries: ${packageCatalog?.counts?.entries ?? 'n/a'}`,
    `- Commentary entries: ${packageCatalog?.counts?.commentary_entries ?? 'n/a'}`,
    `- Dictionary entries: ${packageCatalog?.counts?.dictionary_entries ?? 'n/a'}`,
@@ -201,6 +203,7 @@ function countsLabel(packageCatalog) {
    counts.verses != null ? `${counts.verses} verses` : null,
    counts.strongs_links != null ? `${counts.strongs_links} Strong's links` : null,
    counts.footnotes != null ? `${counts.footnotes} footnotes` : null,
+    counts.study_notes != null ? `${counts.study_notes} study notes` : null,
    counts.commentary_entries != null ? `${counts.commentary_entries} commentary entries` : null,
    counts.dictionary_entries != null ? `${counts.dictionary_entries} dictionary entries` : null,
    counts.entries != null && counts.commentary_entries == null && counts.dictionary_entries == null ? `${counts.entries} entries` : null,
@@ -44,8 +44,10 @@ const usfmEntries = zip
 const verses = []
 const strongsLinks = []
 const footnotes = []
+const studyNotes = []
 let tokenId = 1
 let footnoteId = 1
+let studyNoteId = 1

 for (const entry of usfmEntries) {
  const filePath = path.join(extractDir, entry.entryName)
@@ -53,67 +55,34 @@ for (const entry of usfmEntries) {
  const bookId = /\\id\s+([A-Z0-9]+)/.exec(usfm)?.[1]
  const bookTitle = /\\toc2\s+(.+)/.exec(usfm)?.[1]?.trim() ?? bookId
  let chapter = null
+  let currentVerse = null

  for (const line of usfm.split(/\r?\n/)) {
    const chapterMatch = /^\\c\s+(\d+)/.exec(line)
    if (chapterMatch) {
+      flushVerse(bookId, bookTitle, currentVerse)
+      currentVerse = null
      chapter = Number(chapterMatch[1])
      continue
    }

    const verseMatch = /^\\v\s+(\d+)\s+([\s\S]+)/.exec(line)
-    if (!verseMatch || chapter === null || !bookId) {
+    if (verseMatch && chapter !== null && bookId) {
+      flushVerse(bookId, bookTitle, currentVerse)
+      currentVerse = {
+        chapter,
+        verseNumber: Number(verseMatch[1]),
+        raw: verseMatch[2],
+      }
      continue
    }

-    const verseNumber = Number(verseMatch[1])
-    const raw = verseMatch[2]
-    const verseId = `${bookId}.${chapter}.${verseNumber}`
-    const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
-    const notes = extractFootnotes(raw)
-    const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))
-
-    verses.push({
-      id: verseId,
-      translation_id: manifest.id,
-      book_id: bookId,
-      book: bookTitle,
-      chapter,
-      verse: verseNumber,
-      reference: `${bookTitle} ${chapter}:${verseNumber}`,
-      text,
-    })
-
-    for (const [index, link] of links.entries()) {
-      strongsLinks.push({
-        id: tokenId++,
-        verse_id: verseId,
-        translation_id: manifest.id,
-        book_id: bookId,
-        chapter,
-        verse: verseNumber,
-        position: index + 1,
-        surface: cleanUsfmText(link[1]),
-        strong_raw: link[2],
-        strong: normalizeStrong(link[2]),
-      })
-    }
-
-    for (const [index, note] of notes.entries()) {
-      footnotes.push({
-        id: footnoteId++,
-        verse_id: verseId,
-        translation_id: manifest.id,
-        book_id: bookId,
-        chapter,
-        verse: verseNumber,
-        note_index: index + 1,
-        caller: note.caller,
-        reference: note.reference || `${chapter}.${verseNumber}`,
-        text: note.text,
-      })
+    if (currentVerse && isVerseContinuation(line)) {
+      currentVerse.raw += ` ${line}`
    }
  }
+
+  flushVerse(bookId, bookTitle, currentVerse)
 }

 await mkdir(outputDir, { recursive: true })
@@ -132,10 +101,16 @@ await writeFile(
  `${footnotes.map((note) => JSON.stringify(note)).join('\n')}\n`,
  'utf8',
 )
+await writeFile(
+  path.join(outputDir, 'study-notes.jsonl'),
+  `${studyNotes.map((note) => JSON.stringify(note)).join('\n')}\n`,
+  'utf8',
+)

 const versesPackage = await packageFile(outputDir, 'verses.jsonl')
 const strongsPackage = await packageFile(outputDir, 'strongs-links.jsonl')
 const footnotesPackage = await packageFile(outputDir, 'footnotes.jsonl')
+const studyNotesPackage = await packageFile(outputDir, 'study-notes.jsonl')

 const catalog = {
  schema_version: 'librebible.resource-catalog.v1',
@@ -165,11 +140,13 @@ const catalog = {
    verses: verses.length,
    strongs_links: strongsLinks.length,
    footnotes: footnotes.length,
+    study_notes: studyNotes.length,
  },
  files: {
    verses_jsonl: versesPackage,
    strongs_jsonl: strongsPackage,
    footnotes_jsonl: footnotesPackage,
+    study_notes_jsonl: studyNotesPackage,
  },
 }

@@ -209,6 +186,7 @@ await writeJson(path.join(root, 'packages', 'json', 'catalog.json'), {
 console.log(`${manifest.id}: imported ${catalog.counts.verses} verses`)
 console.log(`${manifest.id}: imported ${catalog.counts.strongs_links} Strong's links`)
 console.log(`${manifest.id}: imported ${catalog.counts.footnotes} footnotes`)
+console.log(`${manifest.id}: imported ${catalog.counts.study_notes} study notes`)

 function extractFootnotes(raw) {
  return [...raw.matchAll(/\\f\s+([\s\S]*?)\\f\*/g)]
@@ -216,16 +194,20 @@ function extractFootnotes(raw) {
      const content = match[1]
      const caller = /^([+\w-]+)/.exec(content.trim())?.[1] ?? '+'
      const reference = /\\fr\s+([^\\]+)/.exec(content)?.[1]?.trim() ?? ''
+      const label = /\\fl\s+([^\\]+)/.exec(content)?.[1]?.trim() ?? 'Footnote'
      const textSource = content
        .replace(/^([+\w-]+)/, '')
        .replace(/\\fr\s+[^\\]+/, '')
+        .replace(/\\fl\s+[^\\]+/, '')
      return {
        caller,
        reference,
+        label,
+        kind: /study note/i.test(label) ? 'study_note' : 'translator_note',
        text: cleanUsfmText(textSource),
      }
    })
-    .filter((note) => note.text.length > 0)
+    .filter((note) => note.text.length > 0 && note.text !== '[[EMPTY]]')
 }

 async function packageFile(outputDir, fileName) {
@@ -237,3 +219,69 @@ async function packageFile(outputDir, fileName) {
    sha256: await sha256File(filePath),
  }
 }
+
+function flushVerse(bookId, bookTitle, currentVerse) {
+  if (!bookId || !currentVerse) return
+
+  const { chapter, verseNumber, raw } = currentVerse
+  const verseId = `${bookId}.${chapter}.${verseNumber}`
+  const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
+  const notes = extractFootnotes(raw)
+  const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))
+
+  verses.push({
+    id: verseId,
+    translation_id: manifest.id,
+    book_id: bookId,
+    book: bookTitle,
+    chapter,
+    verse: verseNumber,
+    reference: `${bookTitle} ${chapter}:${verseNumber}`,
+    text,
+  })
+
+  for (const [index, link] of links.entries()) {
+    strongsLinks.push({
+      id: tokenId++,
+      verse_id: verseId,
+      translation_id: manifest.id,
+      book_id: bookId,
+      chapter,
+      verse: verseNumber,
+      position: index + 1,
+      surface: cleanUsfmText(link[1]),
+      strong_raw: link[2],
+      strong: normalizeStrong(link[2]),
+    })
+  }
+
+  let verseFootnoteIndex = 1
+  let verseStudyNoteIndex = 1
+
+  for (const note of notes) {
+    const target = note.kind === 'study_note' ? studyNotes : footnotes
+    const id = note.kind === 'study_note' ? studyNoteId++ : footnoteId++
+    const noteIndex =
+      note.kind === 'study_note' ? verseStudyNoteIndex++ : verseFootnoteIndex++
+    target.push({
+      id,
+      verse_id: verseId,
+      translation_id: manifest.id,
+      book_id: bookId,
+      chapter,
+      verse: verseNumber,
+      note_index: noteIndex,
+      note_type: note.kind,
+      label: note.label,
+      caller: note.caller,
+      reference: note.reference || `${chapter}.${verseNumber}`,
+      text: note.text,
+    })
+  }
+}
+
+function isVerseContinuation(line) {
+  if (!line.trim()) return false
+  if (/^\\(id|h|toc\d?|mt\d?|ms\d?|s\d?|r|rem)\b/.test(line)) return false
+  return !/^\\c\s+\d+/.test(line)
+}