Add commentary and dictionary resources
This commit is contained in:
@@ -0,0 +1,254 @@
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
|
||||
|
||||
const manifestPath = process.argv[2]
|
||||
if (!manifestPath) {
|
||||
console.error('Usage: node scripts/import-mhcc.js sources/<manifest>.json')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const root = process.cwd()
|
||||
const manifest = await readJson(path.resolve(root, manifestPath))
|
||||
const sourcePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
|
||||
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
||||
|
||||
const bibleBooks = [
|
||||
'Genesis',
|
||||
'Exodus',
|
||||
'Leviticus',
|
||||
'Numbers',
|
||||
'Deuteronomy',
|
||||
'Joshua',
|
||||
'Judges',
|
||||
'Ruth',
|
||||
'1 Samuel',
|
||||
'2 Samuel',
|
||||
'1 Kings',
|
||||
'2 Kings',
|
||||
'1 Chronicles',
|
||||
'2 Chronicles',
|
||||
'Ezra',
|
||||
'Nehemiah',
|
||||
'Esther',
|
||||
'Job',
|
||||
'Psalms',
|
||||
'Proverbs',
|
||||
'Ecclesiastes',
|
||||
'Song of Solomon',
|
||||
'Isaiah',
|
||||
'Jeremiah',
|
||||
'Lamentations',
|
||||
'Ezekiel',
|
||||
'Daniel',
|
||||
'Hosea',
|
||||
'Joel',
|
||||
'Amos',
|
||||
'Obadiah',
|
||||
'Jonah',
|
||||
'Micah',
|
||||
'Nahum',
|
||||
'Habakkuk',
|
||||
'Zephaniah',
|
||||
'Haggai',
|
||||
'Zechariah',
|
||||
'Malachi',
|
||||
'Matthew',
|
||||
'Mark',
|
||||
'Luke',
|
||||
'John',
|
||||
'Acts',
|
||||
'Romans',
|
||||
'1 Corinthians',
|
||||
'2 Corinthians',
|
||||
'Galatians',
|
||||
'Ephesians',
|
||||
'Philippians',
|
||||
'Colossians',
|
||||
'1 Thessalonians',
|
||||
'2 Thessalonians',
|
||||
'1 Timothy',
|
||||
'2 Timothy',
|
||||
'Titus',
|
||||
'Philemon',
|
||||
'Hebrews',
|
||||
'James',
|
||||
'1 Peter',
|
||||
'2 Peter',
|
||||
'1 John',
|
||||
'2 John',
|
||||
'3 John',
|
||||
'Jude',
|
||||
'Revelation',
|
||||
]
|
||||
const bibleBookSet = new Set(bibleBooks)
|
||||
|
||||
await downloadFile(manifest.source.download_url, sourcePath)
|
||||
const sourceSha = await sha256File(sourcePath)
|
||||
if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
||||
throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`)
|
||||
}
|
||||
|
||||
const source = await readFile(sourcePath, 'utf8')
|
||||
const entries = parseCommentary(source)
|
||||
|
||||
await mkdir(packageDir, { recursive: true })
|
||||
await writeJsonl(path.join(packageDir, 'commentary.jsonl'), entries)
|
||||
|
||||
const files = {
|
||||
commentary_jsonl: await fileInfo(path.join(packageDir, 'commentary.jsonl')),
|
||||
}
|
||||
|
||||
await writeJson(path.join(packageDir, 'catalog.json'), {
|
||||
schema_version: 'librebible.resource-catalog.v1',
|
||||
project: manifest.project_name ?? 'LibreBible',
|
||||
id: manifest.id,
|
||||
resource_type: manifest.resource_type,
|
||||
title: manifest.title,
|
||||
short_title: manifest.short_title,
|
||||
abbreviation: manifest.abbreviation,
|
||||
alternate_ids: manifest.alternate_ids ?? [],
|
||||
language: manifest.language,
|
||||
script: manifest.script,
|
||||
canon: manifest.canon,
|
||||
contributors: manifest.contributors ?? [],
|
||||
features: manifest.features ?? [],
|
||||
attachments: manifest.attachments ?? {},
|
||||
source: manifest.source,
|
||||
license: manifest.license,
|
||||
catalog_display: manifest.catalog_display,
|
||||
importer: manifest.importer,
|
||||
source_manifest_file: path.basename(manifestPath),
|
||||
generated_at: new Date().toISOString(),
|
||||
source_sha256: sourceSha,
|
||||
checks: manifest.checks,
|
||||
counts: {
|
||||
entries: entries.length,
|
||||
commentary_entries: entries.length,
|
||||
books: new Set(entries.map((entry) => entry.book)).size,
|
||||
},
|
||||
files,
|
||||
})
|
||||
|
||||
console.log(`${manifest.id}: imported ${entries.length} commentary entries`)
|
||||
|
||||
function parseCommentary(source) {
|
||||
const lines = source.replace(/\r\n/g, '\n').split('\n')
|
||||
const entries = []
|
||||
let currentBook = null
|
||||
let currentChapter = null
|
||||
let currentRange = null
|
||||
let buffer = []
|
||||
|
||||
function flush() {
|
||||
if (!currentBook || !currentChapter || !currentRange || buffer.length === 0) return
|
||||
const text = cleanParagraph(buffer.join(' '))
|
||||
if (!text) return
|
||||
entries.push({
|
||||
id: `mhcc:${slug(currentBook)}:${currentChapter}:${currentRange.start}-${currentRange.end}`,
|
||||
resource_id: 'matthew-henry-concise-ccel',
|
||||
source: 'Matthew Henry Concise Commentary',
|
||||
book: currentBook,
|
||||
chapter: currentChapter,
|
||||
verse_start: currentRange.start,
|
||||
verse_end: currentRange.end,
|
||||
reference: `${currentBook} ${currentChapter}:${currentRange.start}${currentRange.end !== currentRange.start ? `-${currentRange.end}` : ''}`,
|
||||
title: `${currentBook} ${currentChapter}:${currentRange.start}${currentRange.end !== currentRange.start ? `-${currentRange.end}` : ''}`,
|
||||
text,
|
||||
})
|
||||
}
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.trim()
|
||||
if (!line || /^_+$/.test(line) || line === 'Chapter Outline') continue
|
||||
|
||||
if (bibleBookSet.has(line)) {
|
||||
flush()
|
||||
currentBook = line
|
||||
currentChapter = null
|
||||
currentRange = null
|
||||
buffer = []
|
||||
continue
|
||||
}
|
||||
|
||||
const chapterMatch = /^Chapter\s+(\d+)$/i.exec(line)
|
||||
if (chapterMatch && currentBook) {
|
||||
flush()
|
||||
currentChapter = Number(chapterMatch[1])
|
||||
currentRange = null
|
||||
buffer = []
|
||||
continue
|
||||
}
|
||||
|
||||
const range = parseVerseRange(line)
|
||||
if (range && currentBook && currentChapter) {
|
||||
flush()
|
||||
currentRange = range
|
||||
buffer = []
|
||||
continue
|
||||
}
|
||||
|
||||
const inlineRange = parseInlineReferenceRange(line, currentChapter)
|
||||
if (inlineRange && currentBook && currentChapter) {
|
||||
flush()
|
||||
currentRange = inlineRange.range
|
||||
buffer = inlineRange.text ? [inlineRange.text] : []
|
||||
continue
|
||||
}
|
||||
|
||||
if (currentBook && currentChapter && currentRange) {
|
||||
buffer.push(line)
|
||||
}
|
||||
}
|
||||
|
||||
flush()
|
||||
return entries
|
||||
}
|
||||
|
||||
function parseVerseRange(line) {
|
||||
const match = /^Verses?\s+(.+)$/i.exec(line)
|
||||
if (!match) return null
|
||||
const numbers = [...match[1].matchAll(/\d+/g)].map((item) => Number(item[0]))
|
||||
if (!numbers.length) return null
|
||||
return {
|
||||
start: numbers[0],
|
||||
end: numbers[numbers.length - 1],
|
||||
}
|
||||
}
|
||||
|
||||
function parseInlineReferenceRange(line, currentChapter) {
|
||||
const match = /^(?:[1-3]\s*)?[A-Z][A-Za-z.]+\s+(\d+):(\d+)(?:-(\d+))?\s*(.*)$/.exec(line)
|
||||
if (!match) return null
|
||||
const chapter = Number(match[1])
|
||||
if (chapter !== currentChapter) return null
|
||||
const start = Number(match[2])
|
||||
const end = match[3] ? Number(match[3]) : start
|
||||
return {
|
||||
range: { start, end },
|
||||
text: match[4]?.trim() ?? '',
|
||||
}
|
||||
}
|
||||
|
||||
function cleanParagraph(value) {
|
||||
return value
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/\s+([,.;:?!])/g, '$1')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function slug(value) {
|
||||
return value.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
||||
}
|
||||
|
||||
async function writeJsonl(filePath, rows) {
|
||||
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
||||
}
|
||||
|
||||
async function fileInfo(filePath) {
|
||||
const data = await readFile(filePath)
|
||||
return {
|
||||
path: path.basename(filePath),
|
||||
bytes: data.length,
|
||||
sha256: await sha256File(filePath),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
|
||||
|
||||
const manifestPath = process.argv[2]
|
||||
if (!manifestPath) {
|
||||
console.error('Usage: node scripts/import-neuu-dictionaries.js sources/<manifest>.json')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const root = process.cwd()
|
||||
const manifest = await readJson(path.resolve(root, manifestPath))
|
||||
const indexPath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
|
||||
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
||||
|
||||
await downloadFile(manifest.source.download_url, indexPath)
|
||||
const indexSha = await sha256File(indexPath)
|
||||
if (indexSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
||||
throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${indexSha}`)
|
||||
}
|
||||
|
||||
const index = await readJson(indexPath)
|
||||
const entries = []
|
||||
const combinedHash = createHash('sha256')
|
||||
|
||||
for (const file of index.files) {
|
||||
const url = new URL(file, manifest.source.download_url).toString()
|
||||
const data = await fetchJson(url)
|
||||
combinedHash.update(JSON.stringify(data))
|
||||
|
||||
for (const [key, entry] of Object.entries(data)) {
|
||||
entries.push({
|
||||
id: `easton-smith:${entry.slug ?? slug(key)}`,
|
||||
resource_id: manifest.id,
|
||||
term: entry.name ?? key,
|
||||
slug: entry.slug ?? slug(key),
|
||||
definitions: (entry.definitions ?? []).map((definition) => ({
|
||||
source: definition.source,
|
||||
source_name: index.source_names?.[definition.source] ?? definition.source,
|
||||
text: cleanText(definition.text),
|
||||
})),
|
||||
scripture_refs: entry.scripture_refs ?? [],
|
||||
sources: entry.sources ?? [],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
entries.sort((a, b) => a.term.localeCompare(b.term))
|
||||
|
||||
await mkdir(packageDir, { recursive: true })
|
||||
await writeJsonl(path.join(packageDir, 'entries.jsonl'), entries)
|
||||
|
||||
const sourceCounts = {}
|
||||
for (const entry of entries) {
|
||||
for (const source of entry.sources) {
|
||||
sourceCounts[source] = (sourceCounts[source] ?? 0) + 1
|
||||
}
|
||||
}
|
||||
|
||||
const files = {
|
||||
entries_jsonl: await fileInfo(path.join(packageDir, 'entries.jsonl')),
|
||||
}
|
||||
|
||||
await writeJson(path.join(packageDir, 'catalog.json'), {
|
||||
schema_version: 'librebible.resource-catalog.v1',
|
||||
project: manifest.project_name ?? 'LibreBible',
|
||||
id: manifest.id,
|
||||
resource_type: manifest.resource_type,
|
||||
title: manifest.title,
|
||||
short_title: manifest.short_title,
|
||||
abbreviation: manifest.abbreviation,
|
||||
alternate_ids: manifest.alternate_ids ?? [],
|
||||
language: manifest.language,
|
||||
script: manifest.script,
|
||||
canon: manifest.canon,
|
||||
contributors: manifest.contributors ?? [],
|
||||
features: manifest.features ?? [],
|
||||
attachments: manifest.attachments ?? {},
|
||||
source: manifest.source,
|
||||
license: manifest.license,
|
||||
catalog_display: manifest.catalog_display,
|
||||
importer: manifest.importer,
|
||||
source_manifest_file: path.basename(manifestPath),
|
||||
generated_at: new Date().toISOString(),
|
||||
source_sha256: indexSha,
|
||||
combined_source_sha256: combinedHash.digest('hex'),
|
||||
checks: manifest.checks,
|
||||
counts: {
|
||||
entries: entries.length,
|
||||
dictionary_entries: entries.length,
|
||||
scripture_refs: entries.reduce((total, entry) => total + entry.scripture_refs.length, 0),
|
||||
source_entries: sourceCounts,
|
||||
},
|
||||
files,
|
||||
})
|
||||
|
||||
console.log(`${manifest.id}: imported ${entries.length} dictionary entries`)
|
||||
|
||||
async function fetchJson(url) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) throw new Error(`Unable to load ${url}: ${response.status} ${response.statusText}`)
|
||||
return response.json()
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return String(value ?? '')
|
||||
.replace(/“/g, '"')
|
||||
.replace(/â€/g, '"')
|
||||
.replace(/’/g, "'")
|
||||
.replace(/‘/g, "'")
|
||||
.replace(/—/g, '-')
|
||||
.replace(/–/g, '-')
|
||||
.replace(/…/g, '...')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function slug(value) {
|
||||
return String(value).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
||||
}
|
||||
|
||||
async function writeJsonl(filePath, rows) {
|
||||
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
||||
}
|
||||
|
||||
async function fileInfo(filePath) {
|
||||
const data = await readFile(filePath)
|
||||
return {
|
||||
path: path.basename(filePath),
|
||||
bytes: data.length,
|
||||
sha256: await sha256File(filePath),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user