135 lines
4.3 KiB
JavaScript
135 lines
4.3 KiB
JavaScript
import { createHash } from 'node:crypto'
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
|
|
|
|
const manifestPath = process.argv[2]
|
|
if (!manifestPath) {
|
|
console.error('Usage: node scripts/import-neuu-dictionaries.js sources/<manifest>.json')
|
|
process.exit(1)
|
|
}
|
|
|
|
const root = process.cwd()
|
|
const manifest = await readJson(path.resolve(root, manifestPath))
|
|
const indexPath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
|
|
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
|
|
|
await downloadFile(manifest.source.download_url, indexPath)
|
|
const indexSha = await sha256File(indexPath)
|
|
if (indexSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
|
throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${indexSha}`)
|
|
}
|
|
|
|
const index = await readJson(indexPath)
|
|
const entries = []
|
|
const combinedHash = createHash('sha256')
|
|
|
|
for (const file of index.files) {
|
|
const url = new URL(file, manifest.source.download_url).toString()
|
|
const data = await fetchJson(url)
|
|
combinedHash.update(JSON.stringify(data))
|
|
|
|
for (const [key, entry] of Object.entries(data)) {
|
|
entries.push({
|
|
id: `easton-smith:${entry.slug ?? slug(key)}`,
|
|
resource_id: manifest.id,
|
|
term: entry.name ?? key,
|
|
slug: entry.slug ?? slug(key),
|
|
definitions: (entry.definitions ?? []).map((definition) => ({
|
|
source: definition.source,
|
|
source_name: index.source_names?.[definition.source] ?? definition.source,
|
|
text: cleanText(definition.text),
|
|
})),
|
|
scripture_refs: entry.scripture_refs ?? [],
|
|
sources: entry.sources ?? [],
|
|
})
|
|
}
|
|
}
|
|
|
|
entries.sort((a, b) => a.term.localeCompare(b.term))
|
|
|
|
await mkdir(packageDir, { recursive: true })
|
|
await writeJsonl(path.join(packageDir, 'entries.jsonl'), entries)
|
|
|
|
const sourceCounts = {}
|
|
for (const entry of entries) {
|
|
for (const source of entry.sources) {
|
|
sourceCounts[source] = (sourceCounts[source] ?? 0) + 1
|
|
}
|
|
}
|
|
|
|
const files = {
|
|
entries_jsonl: await fileInfo(path.join(packageDir, 'entries.jsonl')),
|
|
}
|
|
|
|
await writeJson(path.join(packageDir, 'catalog.json'), {
|
|
schema_version: 'librebible.resource-catalog.v1',
|
|
project: manifest.project_name ?? 'LibreBible',
|
|
id: manifest.id,
|
|
resource_type: manifest.resource_type,
|
|
title: manifest.title,
|
|
short_title: manifest.short_title,
|
|
abbreviation: manifest.abbreviation,
|
|
alternate_ids: manifest.alternate_ids ?? [],
|
|
language: manifest.language,
|
|
script: manifest.script,
|
|
canon: manifest.canon,
|
|
contributors: manifest.contributors ?? [],
|
|
features: manifest.features ?? [],
|
|
attachments: manifest.attachments ?? {},
|
|
source: manifest.source,
|
|
license: manifest.license,
|
|
catalog_display: manifest.catalog_display,
|
|
importer: manifest.importer,
|
|
source_manifest_file: path.basename(manifestPath),
|
|
generated_at: new Date().toISOString(),
|
|
source_sha256: indexSha,
|
|
combined_source_sha256: combinedHash.digest('hex'),
|
|
checks: manifest.checks,
|
|
counts: {
|
|
entries: entries.length,
|
|
dictionary_entries: entries.length,
|
|
scripture_refs: entries.reduce((total, entry) => total + entry.scripture_refs.length, 0),
|
|
source_entries: sourceCounts,
|
|
},
|
|
files,
|
|
})
|
|
|
|
console.log(`${manifest.id}: imported ${entries.length} dictionary entries`)
|
|
|
|
async function fetchJson(url) {
|
|
const response = await fetch(url)
|
|
if (!response.ok) throw new Error(`Unable to load ${url}: ${response.status} ${response.statusText}`)
|
|
return response.json()
|
|
}
|
|
|
|
function cleanText(value) {
|
|
return String(value ?? '')
|
|
.replace(/“/g, '"')
|
|
.replace(/â€/g, '"')
|
|
.replace(/’/g, "'")
|
|
.replace(/‘/g, "'")
|
|
.replace(/—/g, '-')
|
|
.replace(/–/g, '-')
|
|
.replace(/…/g, '...')
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
}
|
|
|
|
function slug(value) {
|
|
return String(value).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
|
}
|
|
|
|
async function writeJsonl(filePath, rows) {
|
|
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
|
}
|
|
|
|
async function fileInfo(filePath) {
|
|
const data = await readFile(filePath)
|
|
return {
|
|
path: path.basename(filePath),
|
|
bytes: data.length,
|
|
sha256: await sha256File(filePath),
|
|
}
|
|
}
|