Add commentary and dictionary resources

2026-07-12 10:25:16 -05:00
parent a6ee513e02
commit 652e27b2b9
16 changed files with 11171 additions and 7 deletions
@@ -0,0 +1,134 @@
+import { createHash } from 'node:crypto'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
+
+const manifestPath = process.argv[2]
+if (!manifestPath) {
+  console.error('Usage: node scripts/import-neuu-dictionaries.js sources/<manifest>.json')
+  process.exit(1)
+}
+
+const root = process.cwd()
+const manifest = await readJson(path.resolve(root, manifestPath))
+const indexPath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
+const packageDir = path.join(root, 'packages', 'json', manifest.id)
+
+await downloadFile(manifest.source.download_url, indexPath)
+const indexSha = await sha256File(indexPath)
+if (indexSha !== manifest.checks.expected_sha256.toLowerCase()) {
+  throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${indexSha}`)
+}
+
+const index = await readJson(indexPath)
+const entries = []
+const combinedHash = createHash('sha256')
+
+for (const file of index.files) {
+  const url = new URL(file, manifest.source.download_url).toString()
+  const data = await fetchJson(url)
+  combinedHash.update(JSON.stringify(data))
+
+  for (const [key, entry] of Object.entries(data)) {
+    entries.push({
+      id: `easton-smith:${entry.slug ?? slug(key)}`,
+      resource_id: manifest.id,
+      term: entry.name ?? key,
+      slug: entry.slug ?? slug(key),
+      definitions: (entry.definitions ?? []).map((definition) => ({
+        source: definition.source,
+        source_name: index.source_names?.[definition.source] ?? definition.source,
+        text: cleanText(definition.text),
+      })),
+      scripture_refs: entry.scripture_refs ?? [],
+      sources: entry.sources ?? [],
+    })
+  }
+}
+
+entries.sort((a, b) => a.term.localeCompare(b.term))
+
+await mkdir(packageDir, { recursive: true })
+await writeJsonl(path.join(packageDir, 'entries.jsonl'), entries)
+
+const sourceCounts = {}
+for (const entry of entries) {
+  for (const source of entry.sources) {
+    sourceCounts[source] = (sourceCounts[source] ?? 0) + 1
+  }
+}
+
+const files = {
+  entries_jsonl: await fileInfo(path.join(packageDir, 'entries.jsonl')),
+}
+
+await writeJson(path.join(packageDir, 'catalog.json'), {
+  schema_version: 'librebible.resource-catalog.v1',
+  project: manifest.project_name ?? 'LibreBible',
+  id: manifest.id,
+  resource_type: manifest.resource_type,
+  title: manifest.title,
+  short_title: manifest.short_title,
+  abbreviation: manifest.abbreviation,
+  alternate_ids: manifest.alternate_ids ?? [],
+  language: manifest.language,
+  script: manifest.script,
+  canon: manifest.canon,
+  contributors: manifest.contributors ?? [],
+  features: manifest.features ?? [],
+  attachments: manifest.attachments ?? {},
+  source: manifest.source,
+  license: manifest.license,
+  catalog_display: manifest.catalog_display,
+  importer: manifest.importer,
+  source_manifest_file: path.basename(manifestPath),
+  generated_at: new Date().toISOString(),
+  source_sha256: indexSha,
+  combined_source_sha256: combinedHash.digest('hex'),
+  checks: manifest.checks,
+  counts: {
+    entries: entries.length,
+    dictionary_entries: entries.length,
+    scripture_refs: entries.reduce((total, entry) => total + entry.scripture_refs.length, 0),
+    source_entries: sourceCounts,
+  },
+  files,
+})
+
+console.log(`${manifest.id}: imported ${entries.length} dictionary entries`)
+
+async function fetchJson(url) {
+  const response = await fetch(url)
+  if (!response.ok) throw new Error(`Unable to load ${url}: ${response.status} ${response.statusText}`)
+  return response.json()
+}
+
+function cleanText(value) {
+  return String(value ?? '')
+    .replace(/â€œ/g, '"')
+    .replace(/â€/g, '"')
+    .replace(/â€™/g, "'")
+    .replace(/â€˜/g, "'")
+    .replace(/â€”/g, '-')
+    .replace(/â€“/g, '-')
+    .replace(/â€¦/g, '...')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+function slug(value) {
+  return String(value).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
+}
+
+async function writeJsonl(filePath, rows) {
+  await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
+}
+
+async function fileInfo(filePath) {
+  const data = await readFile(filePath)
+  return {
+    path: path.basename(filePath),
+    bytes: data.length,
+    sha256: await sha256File(filePath),
+  }
+}