Initial Bible data repository

2026-07-11 21:36:31 -05:00
commit a7cbe14eae
14 changed files with 380611 additions and 0 deletions
@@ -0,0 +1,26 @@
+import { readdir } from 'node:fs/promises'
+import path from 'node:path'
+import { downloadFile, readJson, sha256File } from './lib.js'
+
+const root = process.cwd()
+const sourceDir = path.join(root, 'sources')
+const files = (await readdir(sourceDir)).filter((file) => file.endsWith('.json'))
+
+for (const file of files) {
+  const manifestPath = path.join(sourceDir, file)
+  const manifest = await readJson(manifestPath)
+  const cachePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
+
+  await downloadFile(manifest.source.download_url, cachePath)
+  const actual = await sha256File(cachePath)
+  const expected = manifest.checks.expected_sha256?.toLowerCase()
+  const status = expected === actual ? 'unchanged' : 'changed'
+
+  console.log(`${manifest.id}: ${status}`)
+  console.log(`  expected: ${expected}`)
+  console.log(`  actual:   ${actual}`)
+
+  if (status === 'changed') {
+    process.exitCode = 1
+  }
+}
@@ -0,0 +1,159 @@
+import AdmZip from 'adm-zip'
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import {
+  cleanUsfmText,
+  downloadFile,
+  exists,
+  normalizeStrong,
+  readJson,
+  sha256File,
+  writeJson,
+} from './lib.js'
+
+const manifestPath = process.argv[2]
+if (!manifestPath) {
+  throw new Error('Usage: node scripts/import-usfm.js sources/<manifest>.json')
+}
+
+const root = process.cwd()
+const manifest = await readJson(path.resolve(root, manifestPath))
+const archivePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
+const extractDir = path.join(root, 'cache', manifest.id, 'usfm')
+const outputDir = path.join(root, 'packages', 'json', manifest.id)
+
+if (!(await exists(archivePath))) {
+  await downloadFile(manifest.source.download_url, archivePath)
+}
+
+const archiveSha = await sha256File(archivePath)
+if (archiveSha !== manifest.checks.expected_sha256.toLowerCase()) {
+  throw new Error(`Checksum mismatch for ${manifest.id}: ${archiveSha}`)
+}
+
+await rm(extractDir, { recursive: true, force: true })
+await mkdir(extractDir, { recursive: true })
+new AdmZip(archivePath).extractAllTo(extractDir, true)
+
+const zip = new AdmZip(archivePath)
+const usfmEntries = zip
+  .getEntries()
+  .filter((entry) => entry.entryName.toLowerCase().endsWith('.usfm'))
+  .sort((a, b) => a.entryName.localeCompare(b.entryName))
+
+const verses = []
+const strongsLinks = []
+let tokenId = 1
+
+for (const entry of usfmEntries) {
+  const filePath = path.join(extractDir, entry.entryName)
+  const usfm = await readFile(filePath, 'utf8')
+  const bookId = /\\id\s+([A-Z0-9]+)/.exec(usfm)?.[1]
+  const bookTitle = /\\toc2\s+(.+)/.exec(usfm)?.[1]?.trim() ?? bookId
+  let chapter = null
+
+  for (const line of usfm.split(/\r?\n/)) {
+    const chapterMatch = /^\\c\s+(\d+)/.exec(line)
+    if (chapterMatch) {
+      chapter = Number(chapterMatch[1])
+      continue
+    }
+
+    const verseMatch = /^\\v\s+(\d+)\s+([\s\S]+)/.exec(line)
+    if (!verseMatch || chapter === null || !bookId) {
+      continue
+    }
+
+    const verseNumber = Number(verseMatch[1])
+    const raw = verseMatch[2]
+    const verseId = `${bookId}.${chapter}.${verseNumber}`
+    const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
+    const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))
+
+    verses.push({
+      id: verseId,
+      translation_id: manifest.id,
+      book_id: bookId,
+      book: bookTitle,
+      chapter,
+      verse: verseNumber,
+      reference: `${bookTitle} ${chapter}:${verseNumber}`,
+      text,
+    })
+
+    for (const [index, link] of links.entries()) {
+      strongsLinks.push({
+        id: tokenId++,
+        verse_id: verseId,
+        translation_id: manifest.id,
+        book_id: bookId,
+        chapter,
+        verse: verseNumber,
+        position: index + 1,
+        surface: cleanUsfmText(link[1]),
+        strong_raw: link[2],
+        strong: normalizeStrong(link[2]),
+      })
+    }
+  }
+}
+
+await mkdir(outputDir, { recursive: true })
+await writeFile(
+  path.join(outputDir, 'verses.jsonl'),
+  `${verses.map((verse) => JSON.stringify(verse)).join('\n')}\n`,
+  'utf8',
+)
+await writeFile(
+  path.join(outputDir, 'strongs-links.jsonl'),
+  `${strongsLinks.map((link) => JSON.stringify(link)).join('\n')}\n`,
+  'utf8',
+)
+
+const catalog = {
+  id: manifest.id,
+  title: manifest.title,
+  abbreviation: manifest.abbreviation,
+  language: manifest.language,
+  features: manifest.features,
+  source: manifest.source,
+  license: manifest.license,
+  importer: manifest.importer,
+  generated_at: new Date().toISOString(),
+  source_sha256: archiveSha,
+  counts: {
+    books: new Set(verses.map((verse) => verse.book_id)).size,
+    verses: verses.length,
+    strongs_links: strongsLinks.length,
+  },
+  files: {
+    verses_jsonl: {
+      path: 'verses.jsonl',
+      sha256: await sha256File(path.join(outputDir, 'verses.jsonl')),
+    },
+    strongs_jsonl: {
+      path: 'strongs-links.jsonl',
+      sha256: await sha256File(path.join(outputDir, 'strongs-links.jsonl')),
+    },
+  },
+}
+
+await writeJson(path.join(outputDir, 'catalog.json'), catalog)
+await writeJson(path.join(root, 'packages', 'json', 'catalog.json'), {
+  generated_at: catalog.generated_at,
+  resources: [
+    {
+      id: catalog.id,
+      title: catalog.title,
+      abbreviation: catalog.abbreviation,
+      language: catalog.language,
+      features: catalog.features,
+      license: catalog.license,
+      package_path: `packages/json/${catalog.id}/catalog.json`,
+      counts: catalog.counts,
+    },
+  ],
+})
+
+console.log(`${manifest.id}: imported ${catalog.counts.verses} verses`)
+console.log(`${manifest.id}: imported ${catalog.counts.strongs_links} Strong's links`)
@@ -0,0 +1,71 @@
+import { createHash } from 'node:crypto'
+import { createWriteStream } from 'node:fs'
+import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+
+export async function readJson(filePath) {
+  return JSON.parse(await readFile(filePath, 'utf8'))
+}
+
+export async function writeJson(filePath, value) {
+  await mkdir(path.dirname(filePath), { recursive: true })
+  await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8')
+}
+
+export async function exists(filePath) {
+  try {
+    await stat(filePath)
+    return true
+  } catch {
+    return false
+  }
+}
+
+export async function downloadFile(url, filePath) {
+  await mkdir(path.dirname(filePath), { recursive: true })
+  const response = await fetch(url)
+  if (!response.ok || !response.body) {
+    throw new Error(`Download failed ${response.status} ${response.statusText}: ${url}`)
+  }
+
+  const output = createWriteStream(filePath)
+  await new Promise((resolve, reject) => {
+    response.body.pipeTo(
+      new WritableStream({
+        write(chunk) {
+          output.write(Buffer.from(chunk))
+        },
+        close() {
+          output.end(resolve)
+        },
+        abort(reason) {
+          output.destroy(reason)
+          reject(reason)
+        },
+      }),
+    ).catch(reject)
+  })
+}
+
+export async function sha256File(filePath) {
+  const data = await readFile(filePath)
+  return createHash('sha256').update(data).digest('hex')
+}
+
+export function normalizeStrong(value) {
+  const match = /^([GH])0*([0-9]+)$/i.exec(value)
+  return match ? `${match[1].toUpperCase()}${Number(match[2])}` : value.toUpperCase()
+}
+
+export function cleanUsfmText(input) {
+  return input
+    .replace(/\\f [\s\S]*?\\f\*/g, '')
+    .replace(/\\x [\s\S]*?\\x\*/g, '')
+    .replace(/\\add\s+([^\\]+)\\add\*/g, '$1')
+    .replace(/\\nd\s+([\s\S]*?)\\nd\*/g, '$1')
+    .replace(/\\[+\w]+\*/g, '')
+    .replace(/\\[a-z0-9+]+(?:\s+)?/gi, '')
+    .replace(/\s+/g, ' ')
+    .replace(/\s+([,.;:?!])/g, '$1')
+    .trim()
+}