libre-bible-data/scripts/import-mhcc.js

import { mkdir, readFile, writeFile } from 'node:fs/promises'
import path from 'node:path'
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'

const manifestPath = process.argv[2]
if (!manifestPath) {
  console.error('Usage: node scripts/import-mhcc.js sources/<manifest>.json')
  process.exit(1)
}

const root = process.cwd()
const manifest = await readJson(path.resolve(root, manifestPath))
const sourcePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
const packageDir = path.join(root, 'packages', 'json', manifest.id)

const bibleBooks = [
  'Genesis',
  'Exodus',
  'Leviticus',
  'Numbers',
  'Deuteronomy',
  'Joshua',
  'Judges',
  'Ruth',
  '1 Samuel',
  '2 Samuel',
  '1 Kings',
  '2 Kings',
  '1 Chronicles',
  '2 Chronicles',
  'Ezra',
  'Nehemiah',
  'Esther',
  'Job',
  'Psalms',
  'Proverbs',
  'Ecclesiastes',
  'Song of Solomon',
  'Isaiah',
  'Jeremiah',
  'Lamentations',
  'Ezekiel',
  'Daniel',
  'Hosea',
  'Joel',
  'Amos',
  'Obadiah',
  'Jonah',
  'Micah',
  'Nahum',
  'Habakkuk',
  'Zephaniah',
  'Haggai',
  'Zechariah',
  'Malachi',
  'Matthew',
  'Mark',
  'Luke',
  'John',
  'Acts',
  'Romans',
  '1 Corinthians',
  '2 Corinthians',
  'Galatians',
  'Ephesians',
  'Philippians',
  'Colossians',
  '1 Thessalonians',
  '2 Thessalonians',
  '1 Timothy',
  '2 Timothy',
  'Titus',
  'Philemon',
  'Hebrews',
  'James',
  '1 Peter',
  '2 Peter',
  '1 John',
  '2 John',
  '3 John',
  'Jude',
  'Revelation',
]
const bibleBookSet = new Set(bibleBooks)

await downloadFile(manifest.source.download_url, sourcePath)
const sourceSha = await sha256File(sourcePath)
if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) {
  throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`)
}

const source = await readFile(sourcePath, 'utf8')
const entries = parseCommentary(source)

await mkdir(packageDir, { recursive: true })
await writeJsonl(path.join(packageDir, 'commentary.jsonl'), entries)

const files = {
  commentary_jsonl: await fileInfo(path.join(packageDir, 'commentary.jsonl')),
}

await writeJson(path.join(packageDir, 'catalog.json'), {
  schema_version: 'librebible.resource-catalog.v1',
  project: manifest.project_name ?? 'LibreBible',
  id: manifest.id,
  resource_type: manifest.resource_type,
  title: manifest.title,
  short_title: manifest.short_title,
  abbreviation: manifest.abbreviation,
  alternate_ids: manifest.alternate_ids ?? [],
  language: manifest.language,
  script: manifest.script,
  canon: manifest.canon,
  contributors: manifest.contributors ?? [],
  features: manifest.features ?? [],
  attachments: manifest.attachments ?? {},
  source: manifest.source,
  license: manifest.license,
  catalog_display: manifest.catalog_display,
  importer: manifest.importer,
  source_manifest_file: path.basename(manifestPath),
  generated_at: new Date().toISOString(),
  source_sha256: sourceSha,
  checks: manifest.checks,
  counts: {
    entries: entries.length,
    commentary_entries: entries.length,
    books: new Set(entries.map((entry) => entry.book)).size,
  },
  files,
})

console.log(`${manifest.id}: imported ${entries.length} commentary entries`)

function parseCommentary(source) {
  const lines = source.replace(/\r\n/g, '\n').split('\n')
  const entries = []
  let currentBook = null
  let currentChapter = null
  let currentRange = null
  let buffer = []

  function flush() {
    if (!currentBook || !currentChapter || !currentRange || buffer.length === 0) return
    const text = cleanParagraph(buffer.join(' '))
    if (!text) return
    entries.push({
      id: `mhcc:${slug(currentBook)}:${currentChapter}:${currentRange.start}-${currentRange.end}`,
      resource_id: 'matthew-henry-concise-ccel',
      source: 'Matthew Henry Concise Commentary',
      book: currentBook,
      chapter: currentChapter,
      verse_start: currentRange.start,
      verse_end: currentRange.end,
      reference: `${currentBook} ${currentChapter}:${currentRange.start}${currentRange.end !== currentRange.start ? `-${currentRange.end}` : ''}`,
      title: `${currentBook} ${currentChapter}:${currentRange.start}${currentRange.end !== currentRange.start ? `-${currentRange.end}` : ''}`,
      text,
    })
  }

  for (const rawLine of lines) {
    const line = rawLine.trim()
    if (!line || /^_+$/.test(line) || line === 'Chapter Outline') continue

    if (bibleBookSet.has(line)) {
      flush()
      currentBook = line
      currentChapter = null
      currentRange = null
      buffer = []
      continue
    }

    const chapterMatch = /^Chapter\s+(\d+)$/i.exec(line)
    if (chapterMatch && currentBook) {
      flush()
      currentChapter = Number(chapterMatch[1])
      currentRange = null
      buffer = []
      continue
    }

    const range = parseVerseRange(line)
    if (range && currentBook && currentChapter) {
      flush()
      currentRange = range
      buffer = []
      continue
    }

    const inlineRange = parseInlineReferenceRange(line, currentChapter)
    if (inlineRange && currentBook && currentChapter) {
      flush()
      currentRange = inlineRange.range
      buffer = inlineRange.text ? [inlineRange.text] : []
      continue
    }

    if (currentBook && currentChapter && currentRange) {
      buffer.push(line)
    }
  }

  flush()
  return entries
}

function parseVerseRange(line) {
  const match = /^Verses?\s+(.+)$/i.exec(line)
  if (!match) return null
  const numbers = [...match[1].matchAll(/\d+/g)].map((item) => Number(item[0]))
  if (!numbers.length) return null
  return {
    start: numbers[0],
    end: numbers[numbers.length - 1],
  }
}

function parseInlineReferenceRange(line, currentChapter) {
  const match = /^(?:[1-3]\s*)?[A-Z][A-Za-z.]+\s+(\d+):(\d+)(?:-(\d+))?\s*(.*)$/.exec(line)
  if (!match) return null
  const chapter = Number(match[1])
  if (chapter !== currentChapter) return null
  const start = Number(match[2])
  const end = match[3] ? Number(match[3]) : start
  return {
    range: { start, end },
    text: match[4]?.trim() ?? '',
  }
}

function cleanParagraph(value) {
  return value
    .replace(/\s+/g, ' ')
    .replace(/\s+([,.;:?!])/g, '$1')
    .trim()
}

function slug(value) {
  return value.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
}

async function writeJsonl(filePath, rows) {
  await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
}

async function fileInfo(filePath) {
  const data = await readFile(filePath)
  return {
    path: path.basename(filePath),
    bytes: data.length,
    sha256: await sha256File(filePath),
  }
}