libre-bible-data/scripts/import-usfm.js

import AdmZip from 'adm-zip'
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
import path from 'node:path'
import {
  cleanUsfmText,
  downloadFile,
  exists,
  normalizeStrong,
  readJson,
  sha256File,
  writeJson,
} from './lib.js'

const manifestPath = process.argv[2]
if (!manifestPath) {
  throw new Error('Usage: node scripts/import-usfm.js sources/<manifest>.json')
}

const root = process.cwd()
const manifest = await readJson(path.resolve(root, manifestPath))
const archivePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
const extractDir = path.join(root, 'cache', manifest.id, 'usfm')
const outputDir = path.join(root, 'packages', 'json', manifest.id)

if (!(await exists(archivePath))) {
  await downloadFile(manifest.source.download_url, archivePath)
}

const archiveSha = await sha256File(archivePath)
if (archiveSha !== manifest.checks.expected_sha256.toLowerCase()) {
  throw new Error(`Checksum mismatch for ${manifest.id}: ${archiveSha}`)
}

await rm(extractDir, { recursive: true, force: true })
await mkdir(extractDir, { recursive: true })
new AdmZip(archivePath).extractAllTo(extractDir, true)

const zip = new AdmZip(archivePath)
const usfmEntries = zip
  .getEntries()
  .filter((entry) => entry.entryName.toLowerCase().endsWith('.usfm'))
  .sort((a, b) => a.entryName.localeCompare(b.entryName))

const verses = []
const strongsLinks = []
let tokenId = 1

for (const entry of usfmEntries) {
  const filePath = path.join(extractDir, entry.entryName)
  const usfm = await readFile(filePath, 'utf8')
  const bookId = /\\id\s+([A-Z0-9]+)/.exec(usfm)?.[1]
  const bookTitle = /\\toc2\s+(.+)/.exec(usfm)?.[1]?.trim() ?? bookId
  let chapter = null

  for (const line of usfm.split(/\r?\n/)) {
    const chapterMatch = /^\\c\s+(\d+)/.exec(line)
    if (chapterMatch) {
      chapter = Number(chapterMatch[1])
      continue
    }

    const verseMatch = /^\\v\s+(\d+)\s+([\s\S]+)/.exec(line)
    if (!verseMatch || chapter === null || !bookId) {
      continue
    }

    const verseNumber = Number(verseMatch[1])
    const raw = verseMatch[2]
    const verseId = `${bookId}.${chapter}.${verseNumber}`
    const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
    const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))

    verses.push({
      id: verseId,
      translation_id: manifest.id,
      book_id: bookId,
      book: bookTitle,
      chapter,
      verse: verseNumber,
      reference: `${bookTitle} ${chapter}:${verseNumber}`,
      text,
    })

    for (const [index, link] of links.entries()) {
      strongsLinks.push({
        id: tokenId++,
        verse_id: verseId,
        translation_id: manifest.id,
        book_id: bookId,
        chapter,
        verse: verseNumber,
        position: index + 1,
        surface: cleanUsfmText(link[1]),
        strong_raw: link[2],
        strong: normalizeStrong(link[2]),
      })
    }
  }
}

await mkdir(outputDir, { recursive: true })
await writeFile(
  path.join(outputDir, 'verses.jsonl'),
  `${verses.map((verse) => JSON.stringify(verse)).join('\n')}\n`,
  'utf8',
)
await writeFile(
  path.join(outputDir, 'strongs-links.jsonl'),
  `${strongsLinks.map((link) => JSON.stringify(link)).join('\n')}\n`,
  'utf8',
)

const versesPackage = await packageFile(outputDir, 'verses.jsonl')
const strongsPackage = await packageFile(outputDir, 'strongs-links.jsonl')

const catalog = {
  schema_version: 'librebible.resource-catalog.v1',
  project: manifest.project_name ?? 'LibreBible',
  id: manifest.id,
  resource_type: manifest.resource_type,
  title: manifest.title,
  short_title: manifest.short_title,
  abbreviation: manifest.abbreviation,
  alternate_ids: manifest.alternate_ids ?? [],
  language: manifest.language,
  script: manifest.script,
  canon: manifest.canon,
  translation: manifest.translation,
  contributors: manifest.contributors ?? [],
  features: manifest.features,
  attachments: manifest.attachments,
  source: manifest.source,
  license: manifest.license,
  catalog_display: manifest.catalog_display,
  importer: manifest.importer,
  generated_at: new Date().toISOString(),
  source_sha256: archiveSha,
  checks: manifest.checks,
  counts: {
    books: new Set(verses.map((verse) => verse.book_id)).size,
    verses: verses.length,
    strongs_links: strongsLinks.length,
  },
  files: {
    verses_jsonl: versesPackage,
    strongs_jsonl: strongsPackage,
  },
}

await writeJson(path.join(outputDir, 'catalog.json'), catalog)
await writeJson(path.join(root, 'packages', 'json', 'catalog.json'), {
  schema_version: 'librebible.index-catalog.v1',
  project: manifest.project_name ?? 'LibreBible',
  generated_at: catalog.generated_at,
  resources: [
    {
      id: catalog.id,
      resource_type: catalog.resource_type,
      title: catalog.title,
      short_title: catalog.short_title,
      abbreviation: catalog.abbreviation,
      alternate_ids: catalog.alternate_ids,
      language: catalog.language,
      script: catalog.script,
      canon: catalog.canon,
      translation: catalog.translation,
      contributors: catalog.contributors,
      features: catalog.features,
      attachments: catalog.attachments,
      source: catalog.source,
      license: catalog.license,
      catalog_display: catalog.catalog_display,
      package_path: `packages/json/${catalog.id}/catalog.json`,
      source_manifest_path: `sources/${path.basename(manifestPath)}`,
      detail_doc_path: `docs/resources/${catalog.id}.md`,
      counts: catalog.counts,
      files: catalog.files,
      checks: catalog.checks,
    },
  ],
})

console.log(`${manifest.id}: imported ${catalog.counts.verses} verses`)
console.log(`${manifest.id}: imported ${catalog.counts.strongs_links} Strong's links`)

async function packageFile(outputDir, fileName) {
  const filePath = path.join(outputDir, fileName)
  const fileStat = await stat(filePath)
  return {
    path: fileName,
    bytes: fileStat.size,
    sha256: await sha256File(filePath),
  }
}