import { mkdir, readFile, writeFile } from 'node:fs/promises' import path from 'node:path' import { downloadFile, readJson, sha256File, writeJson } from './lib.js' const manifestPath = process.argv[2] if (!manifestPath) { console.error('Usage: node scripts/import-mhcc.js sources/.json') process.exit(1) } const root = process.cwd() const manifest = await readJson(path.resolve(root, manifestPath)) const sourcePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url)) const packageDir = path.join(root, 'packages', 'json', manifest.id) const bibleBooks = [ 'Genesis', 'Exodus', 'Leviticus', 'Numbers', 'Deuteronomy', 'Joshua', 'Judges', 'Ruth', '1 Samuel', '2 Samuel', '1 Kings', '2 Kings', '1 Chronicles', '2 Chronicles', 'Ezra', 'Nehemiah', 'Esther', 'Job', 'Psalms', 'Proverbs', 'Ecclesiastes', 'Song of Solomon', 'Isaiah', 'Jeremiah', 'Lamentations', 'Ezekiel', 'Daniel', 'Hosea', 'Joel', 'Amos', 'Obadiah', 'Jonah', 'Micah', 'Nahum', 'Habakkuk', 'Zephaniah', 'Haggai', 'Zechariah', 'Malachi', 'Matthew', 'Mark', 'Luke', 'John', 'Acts', 'Romans', '1 Corinthians', '2 Corinthians', 'Galatians', 'Ephesians', 'Philippians', 'Colossians', '1 Thessalonians', '2 Thessalonians', '1 Timothy', '2 Timothy', 'Titus', 'Philemon', 'Hebrews', 'James', '1 Peter', '2 Peter', '1 John', '2 John', '3 John', 'Jude', 'Revelation', ] const bibleBookSet = new Set(bibleBooks) await downloadFile(manifest.source.download_url, sourcePath) const sourceSha = await sha256File(sourcePath) if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) { throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`) } const source = await readFile(sourcePath, 'utf8') const entries = parseCommentary(source) await mkdir(packageDir, { recursive: true }) await writeJsonl(path.join(packageDir, 'commentary.jsonl'), entries) const files = { commentary_jsonl: await fileInfo(path.join(packageDir, 'commentary.jsonl')), } await writeJson(path.join(packageDir, 'catalog.json'), { schema_version: 'librebible.resource-catalog.v1', project: manifest.project_name ?? 'LibreBible', id: manifest.id, resource_type: manifest.resource_type, title: manifest.title, short_title: manifest.short_title, abbreviation: manifest.abbreviation, alternate_ids: manifest.alternate_ids ?? [], language: manifest.language, script: manifest.script, canon: manifest.canon, contributors: manifest.contributors ?? [], features: manifest.features ?? [], attachments: manifest.attachments ?? {}, source: manifest.source, license: manifest.license, catalog_display: manifest.catalog_display, importer: manifest.importer, source_manifest_file: path.basename(manifestPath), generated_at: new Date().toISOString(), source_sha256: sourceSha, checks: manifest.checks, counts: { entries: entries.length, commentary_entries: entries.length, books: new Set(entries.map((entry) => entry.book)).size, }, files, }) console.log(`${manifest.id}: imported ${entries.length} commentary entries`) function parseCommentary(source) { const lines = source.replace(/\r\n/g, '\n').split('\n') const entries = [] let currentBook = null let currentChapter = null let currentRange = null let buffer = [] function flush() { if (!currentBook || !currentChapter || !currentRange || buffer.length === 0) return const text = cleanParagraph(buffer.join(' ')) if (!text) return entries.push({ id: `mhcc:${slug(currentBook)}:${currentChapter}:${currentRange.start}-${currentRange.end}`, resource_id: 'matthew-henry-concise-ccel', source: 'Matthew Henry Concise Commentary', book: currentBook, chapter: currentChapter, verse_start: currentRange.start, verse_end: currentRange.end, reference: `${currentBook} ${currentChapter}:${currentRange.start}${currentRange.end !== currentRange.start ? `-${currentRange.end}` : ''}`, title: `${currentBook} ${currentChapter}:${currentRange.start}${currentRange.end !== currentRange.start ? `-${currentRange.end}` : ''}`, text, }) } for (const rawLine of lines) { const line = rawLine.trim() if (!line || /^_+$/.test(line) || line === 'Chapter Outline') continue if (bibleBookSet.has(line)) { flush() currentBook = line currentChapter = null currentRange = null buffer = [] continue } const chapterMatch = /^Chapter\s+(\d+)$/i.exec(line) if (chapterMatch && currentBook) { flush() currentChapter = Number(chapterMatch[1]) currentRange = null buffer = [] continue } const range = parseVerseRange(line) if (range && currentBook && currentChapter) { flush() currentRange = range buffer = [] continue } const inlineRange = parseInlineReferenceRange(line, currentChapter) if (inlineRange && currentBook && currentChapter) { flush() currentRange = inlineRange.range buffer = inlineRange.text ? [inlineRange.text] : [] continue } if (currentBook && currentChapter && currentRange) { buffer.push(line) } } flush() return entries } function parseVerseRange(line) { const match = /^Verses?\s+(.+)$/i.exec(line) if (!match) return null const numbers = [...match[1].matchAll(/\d+/g)].map((item) => Number(item[0])) if (!numbers.length) return null return { start: numbers[0], end: numbers[numbers.length - 1], } } function parseInlineReferenceRange(line, currentChapter) { const match = /^(?:[1-3]\s*)?[A-Z][A-Za-z.]+\s+(\d+):(\d+)(?:-(\d+))?\s*(.*)$/.exec(line) if (!match) return null const chapter = Number(match[1]) if (chapter !== currentChapter) return null const start = Number(match[2]) const end = match[3] ? Number(match[3]) : start return { range: { start, end }, text: match[4]?.trim() ?? '', } } function cleanParagraph(value) { return value .replace(/\s+/g, ' ') .replace(/\s+([,.;:?!])/g, '$1') .trim() } function slug(value) { return value.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') } async function writeJsonl(filePath, rows) { await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8') } async function fileInfo(filePath) { const data = await readFile(filePath) return { path: path.basename(filePath), bytes: data.length, sha256: await sha256File(filePath), } }