libre-bible-data/scripts/import-usfm.js

import AdmZip from 'adm-zip'
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
import path from 'node:path'
import {
  cleanUsfmText,
  downloadFile,
  exists,
  normalizeStrong,
  readJson,
  sha256File,
  writeJson,
} from './lib.js'

const manifestPath = process.argv[2]
if (!manifestPath) {
  throw new Error('Usage: node scripts/import-usfm.js sources/<manifest>.json')
}

const root = process.cwd()
const manifest = await readJson(path.resolve(root, manifestPath))
const archivePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
const extractDir = path.join(root, 'cache', manifest.id, 'usfm')
const outputDir = path.join(root, 'packages', 'json', manifest.id)

if (!(await exists(archivePath))) {
  await downloadFile(manifest.source.download_url, archivePath)
}

const archiveSha = await sha256File(archivePath)
if (archiveSha !== manifest.checks.expected_sha256.toLowerCase()) {
  throw new Error(`Checksum mismatch for ${manifest.id}: ${archiveSha}`)
}

await rm(extractDir, { recursive: true, force: true })
await mkdir(extractDir, { recursive: true })
new AdmZip(archivePath).extractAllTo(extractDir, true)

const zip = new AdmZip(archivePath)
const usfmEntries = zip
  .getEntries()
  .filter((entry) => entry.entryName.toLowerCase().endsWith('.usfm'))
  .sort((a, b) => a.entryName.localeCompare(b.entryName))

const verses = []
const strongsLinks = []
let tokenId = 1

for (const entry of usfmEntries) {
  const filePath = path.join(extractDir, entry.entryName)
  const usfm = await readFile(filePath, 'utf8')
  const bookId = /\\id\s+([A-Z0-9]+)/.exec(usfm)?.[1]
  const bookTitle = /\\toc2\s+(.+)/.exec(usfm)?.[1]?.trim() ?? bookId
  let chapter = null

  for (const line of usfm.split(/\r?\n/)) {
    const chapterMatch = /^\\c\s+(\d+)/.exec(line)
    if (chapterMatch) {
      chapter = Number(chapterMatch[1])
      continue
    }

    const verseMatch = /^\\v\s+(\d+)\s+([\s\S]+)/.exec(line)
    if (!verseMatch || chapter === null || !bookId) {
      continue
    }

    const verseNumber = Number(verseMatch[1])
    const raw = verseMatch[2]
    const verseId = `${bookId}.${chapter}.${verseNumber}`
    const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
    const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))

    verses.push({
      id: verseId,
      translation_id: manifest.id,
      book_id: bookId,
      book: bookTitle,
      chapter,
      verse: verseNumber,
      reference: `${bookTitle} ${chapter}:${verseNumber}`,
      text,
    })

    for (const [index, link] of links.entries()) {
      strongsLinks.push({
        id: tokenId++,
        verse_id: verseId,
        translation_id: manifest.id,
        book_id: bookId,
        chapter,
        verse: verseNumber,
        position: index + 1,
        surface: cleanUsfmText(link[1]),
        strong_raw: link[2],
        strong: normalizeStrong(link[2]),
      })
    }
  }
}

await mkdir(outputDir, { recursive: true })
await writeFile(
  path.join(outputDir, 'verses.jsonl'),
  `${verses.map((verse) => JSON.stringify(verse)).join('\n')}\n`,
  'utf8',
)
await writeFile(
  path.join(outputDir, 'strongs-links.jsonl'),
  `${strongsLinks.map((link) => JSON.stringify(link)).join('\n')}\n`,
  'utf8',
)

const catalog = {
  id: manifest.id,
  title: manifest.title,
  abbreviation: manifest.abbreviation,
  language: manifest.language,
  features: manifest.features,
  source: manifest.source,
  license: manifest.license,
  importer: manifest.importer,
  generated_at: new Date().toISOString(),
  source_sha256: archiveSha,
  counts: {
    books: new Set(verses.map((verse) => verse.book_id)).size,
    verses: verses.length,
    strongs_links: strongsLinks.length,
  },
  files: {
    verses_jsonl: {
      path: 'verses.jsonl',
      sha256: await sha256File(path.join(outputDir, 'verses.jsonl')),
    },
    strongs_jsonl: {
      path: 'strongs-links.jsonl',
      sha256: await sha256File(path.join(outputDir, 'strongs-links.jsonl')),
    },
  },
}

await writeJson(path.join(outputDir, 'catalog.json'), catalog)
await writeJson(path.join(root, 'packages', 'json', 'catalog.json'), {
  generated_at: catalog.generated_at,
  resources: [
    {
      id: catalog.id,
      title: catalog.title,
      abbreviation: catalog.abbreviation,
      language: catalog.language,
      features: catalog.features,
      license: catalog.license,
      package_path: `packages/json/${catalog.id}/catalog.json`,
      counts: catalog.counts,
    },
  ],
})

console.log(`${manifest.id}: imported ${catalog.counts.verses} verses`)
console.log(`${manifest.id}: imported ${catalog.counts.strongs_links} Strong's links`)