359 lines
11 KiB
JavaScript
359 lines
11 KiB
JavaScript
import AdmZip from 'adm-zip'
|
|
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
|
|
|
|
const manifestPath = process.argv[2]
|
|
if (!manifestPath) {
|
|
console.error('Usage: node scripts/import-openbible-places.js sources/<manifest>.json')
|
|
process.exit(1)
|
|
}
|
|
|
|
const root = process.cwd()
|
|
const manifest = await readJson(path.resolve(root, manifestPath))
|
|
const zipPath = path.join(root, 'cache', manifest.id, `${manifest.id}.zip`)
|
|
const extractDir = path.join(root, 'cache', manifest.id, 'source')
|
|
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
|
|
|
await downloadFile(manifest.source.download_url, zipPath)
|
|
const sourceSha = await sha256File(zipPath)
|
|
if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
|
throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`)
|
|
}
|
|
|
|
await rm(extractDir, { recursive: true, force: true })
|
|
await mkdir(extractDir, { recursive: true })
|
|
new AdmZip(zipPath).extractAllTo(extractDir, true)
|
|
|
|
const sourceRoot = await findExtractedRoot(extractDir, 'data/ancient.jsonl')
|
|
const ancientRows = await readJsonl(path.join(sourceRoot, 'data', 'ancient.jsonl'))
|
|
const modernRows = await readJsonl(path.join(sourceRoot, 'data', 'modern.jsonl'))
|
|
|
|
const modernLocations = modernRows.map((row) => ({
|
|
id: row.id,
|
|
name: row.friendly_id,
|
|
type: row.type ?? row.class ?? '',
|
|
class: row.class ?? '',
|
|
url_slug: row.url_slug ?? '',
|
|
geometry: row.geometry ?? '',
|
|
land_or_water: row.land_or_water ?? '',
|
|
lon: parseLonLat(row.lonlat).lon,
|
|
lat: parseLonLat(row.lonlat).lat,
|
|
lonlat_type: row.lonlat_type ?? row.precision?.description ?? '',
|
|
precision: row.precision ?? null,
|
|
names: row.names ?? [],
|
|
media: normalizeMedia(row.media),
|
|
source: {
|
|
geometry_credit: row.geometry_credit ?? '',
|
|
coordinates_source: row.coordinates_source ?? null,
|
|
secondary_sources: row.secondary_sources ?? [],
|
|
},
|
|
}))
|
|
|
|
const modernById = new Map(modernLocations.map((location) => [location.id, location]))
|
|
const places = []
|
|
const versePlaces = []
|
|
const placeImages = []
|
|
|
|
for (const row of ancientRows) {
|
|
const best = bestIdentification(row, modernById)
|
|
const place = {
|
|
id: row.id,
|
|
name: row.friendly_id,
|
|
url_slug: row.url_slug ?? '',
|
|
types: row.types ?? [],
|
|
preceding_article: row.preceding_article ?? '',
|
|
comment: row.comment ?? '',
|
|
translation_name_counts: row.translation_name_counts ?? {},
|
|
linked_data: row.linked_data ?? {},
|
|
geometry_credit: row.geometry_credit ?? '',
|
|
geojson_file: row.geojson_file ?? '',
|
|
best_modern_location_id: best?.id ?? '',
|
|
best_modern_location_name: best?.name ?? '',
|
|
lon: best?.lon ?? null,
|
|
lat: best?.lat ?? null,
|
|
confidence: best?.confidence ?? null,
|
|
confidence_label: confidenceLabel(best?.confidence),
|
|
media: normalizeMedia(row.media),
|
|
identifications: (row.identifications ?? []).map((identification) => ({
|
|
id: identification.id,
|
|
description: stripTags(identification.description ?? ''),
|
|
class: identification.class ?? '',
|
|
types: identification.resolutions?.flatMap((resolution) => resolution.types ?? []) ?? [],
|
|
score: identification.score?.time_total ?? identification.score?.vote_total ?? null,
|
|
})),
|
|
}
|
|
places.push(place)
|
|
|
|
const thumbnail = place.media?.thumbnail
|
|
if (thumbnail?.file || thumbnail?.credit_url) {
|
|
placeImages.push({
|
|
id: `${place.id}:thumbnail`,
|
|
place_id: place.id,
|
|
modern_location_id: place.best_modern_location_id,
|
|
kind: 'thumbnail',
|
|
file: thumbnail.file ?? '',
|
|
image_id: thumbnail.image_id ?? '',
|
|
description: stripTags(thumbnail.description ?? ''),
|
|
credit: thumbnail.credit ?? '',
|
|
credit_url: thumbnail.credit_url ?? '',
|
|
placeholder: thumbnail.placeholder ?? '',
|
|
})
|
|
}
|
|
|
|
for (const verse of row.verses ?? []) {
|
|
versePlaces.push({
|
|
id: `${osisToLibreVerseId(verse.osis)}:${row.id}`,
|
|
verse_id: osisToLibreVerseId(verse.osis),
|
|
osis: verse.osis,
|
|
usx: verse.usx ?? '',
|
|
readable: verse.readable ?? '',
|
|
translations: verse.translations ?? [],
|
|
place_id: row.id,
|
|
place_name: row.friendly_id,
|
|
place_types: row.types ?? [],
|
|
modern_location_id: place.best_modern_location_id,
|
|
modern_location_name: place.best_modern_location_name,
|
|
lon: place.lon,
|
|
lat: place.lat,
|
|
confidence: place.confidence,
|
|
confidence_label: place.confidence_label,
|
|
instance_types: verse.instance_types ?? {},
|
|
})
|
|
}
|
|
}
|
|
|
|
places.sort((a, b) => a.name.localeCompare(b.name))
|
|
modernLocations.sort((a, b) => a.name.localeCompare(b.name))
|
|
versePlaces.sort((a, b) => a.verse_id.localeCompare(b.verse_id) || a.place_name.localeCompare(b.place_name))
|
|
placeImages.sort((a, b) => a.place_id.localeCompare(b.place_id))
|
|
|
|
await mkdir(packageDir, { recursive: true })
|
|
await writeJsonl(path.join(packageDir, 'places.jsonl'), places)
|
|
await writeJsonl(path.join(packageDir, 'modern-locations.jsonl'), modernLocations)
|
|
await writeJsonl(path.join(packageDir, 'verse-places.jsonl'), versePlaces)
|
|
await writeJsonl(path.join(packageDir, 'place-images.jsonl'), placeImages)
|
|
|
|
const files = {
|
|
places_jsonl: await fileInfo(path.join(packageDir, 'places.jsonl')),
|
|
modern_locations_jsonl: await fileInfo(path.join(packageDir, 'modern-locations.jsonl')),
|
|
verse_places_jsonl: await fileInfo(path.join(packageDir, 'verse-places.jsonl')),
|
|
place_images_jsonl: await fileInfo(path.join(packageDir, 'place-images.jsonl')),
|
|
}
|
|
|
|
await writeJson(path.join(packageDir, 'catalog.json'), {
|
|
schema_version: 'librebible.resource-catalog.v1',
|
|
project: manifest.project_name ?? 'LibreBible',
|
|
id: manifest.id,
|
|
resource_type: manifest.resource_type,
|
|
title: manifest.title,
|
|
short_title: manifest.short_title,
|
|
abbreviation: manifest.abbreviation,
|
|
alternate_ids: manifest.alternate_ids ?? [],
|
|
language: manifest.language,
|
|
script: manifest.script,
|
|
canon: manifest.canon,
|
|
contributors: manifest.contributors ?? [],
|
|
features: manifest.features ?? [],
|
|
attachments: manifest.attachments ?? {},
|
|
source: manifest.source,
|
|
license: manifest.license,
|
|
catalog_display: manifest.catalog_display,
|
|
importer: manifest.importer,
|
|
source_manifest_file: path.basename(manifestPath),
|
|
generated_at: new Date().toISOString(),
|
|
source_sha256: sourceSha,
|
|
checks: manifest.checks,
|
|
counts: {
|
|
entries: places.length,
|
|
places: places.length,
|
|
modern_locations: modernLocations.length,
|
|
verse_place_links: versePlaces.length,
|
|
verses_with_places: new Set(versePlaces.map((entry) => entry.verse_id)).size,
|
|
place_images: placeImages.length,
|
|
},
|
|
files,
|
|
})
|
|
|
|
console.log(`${manifest.id}: imported ${places.length} places, ${modernLocations.length} modern locations, ${versePlaces.length} verse-place links`)
|
|
|
|
async function findExtractedRoot(baseDir, requiredRelativePath) {
|
|
const entries = await import('node:fs/promises').then((fs) => fs.readdir(baseDir, { withFileTypes: true }))
|
|
for (const entry of entries) {
|
|
if (!entry.isDirectory()) continue
|
|
const candidate = path.join(baseDir, entry.name)
|
|
try {
|
|
await readFile(path.join(candidate, requiredRelativePath), 'utf8')
|
|
return candidate
|
|
} catch {
|
|
// Keep looking.
|
|
}
|
|
}
|
|
throw new Error(`Unable to find extracted source containing ${requiredRelativePath}`)
|
|
}
|
|
|
|
async function readJsonl(filePath) {
|
|
const text = await readFile(filePath, 'utf8')
|
|
return text
|
|
.split('\n')
|
|
.filter((line) => line.trim().length > 0)
|
|
.map((line) => JSON.parse(line))
|
|
}
|
|
|
|
function bestIdentification(row, modernById) {
|
|
const associations = Object.entries(row.modern_associations ?? {})
|
|
.map(([id, value]) => ({
|
|
id,
|
|
name: value.name,
|
|
confidence: Number(value.score ?? 0),
|
|
}))
|
|
.sort((left, right) => right.confidence - left.confidence)
|
|
|
|
for (const association of associations) {
|
|
const modern = modernById.get(association.id)
|
|
if (!modern) continue
|
|
return {
|
|
...modern,
|
|
confidence: association.confidence,
|
|
}
|
|
}
|
|
|
|
for (const identification of row.identifications ?? []) {
|
|
const modern = modernById.get(identification.id)
|
|
if (!modern) continue
|
|
return {
|
|
...modern,
|
|
confidence: identification.score?.time_total ?? identification.score?.vote_total ?? null,
|
|
}
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
function confidenceLabel(score) {
|
|
if (score == null) return 'unknown'
|
|
if (score >= 800) return 'high'
|
|
if (score >= 400) return 'medium'
|
|
if (score > 0) return 'low'
|
|
return 'unknown'
|
|
}
|
|
|
|
function normalizeMedia(media) {
|
|
if (!media) return null
|
|
return {
|
|
thumbnail: media.thumbnail
|
|
? {
|
|
file: media.thumbnail.file ?? '',
|
|
image_id: media.thumbnail.image_id ?? '',
|
|
description: stripTags(media.thumbnail.description ?? ''),
|
|
credit: media.thumbnail.credit ?? '',
|
|
credit_url: media.thumbnail.credit_url ?? '',
|
|
placeholder: media.thumbnail.placeholder ?? '',
|
|
}
|
|
: null,
|
|
}
|
|
}
|
|
|
|
function parseLonLat(value) {
|
|
const [lon, lat] = String(value ?? '').split(',').map((part) => Number(part.trim()))
|
|
return {
|
|
lon: Number.isFinite(lon) ? lon : null,
|
|
lat: Number.isFinite(lat) ? lat : null,
|
|
}
|
|
}
|
|
|
|
function stripTags(value) {
|
|
return String(value).replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim()
|
|
}
|
|
|
|
function osisToLibreVerseId(osis) {
|
|
const match = /^([1-3]?[A-Za-z]+)\.(\d+)\.(\d+)$/.exec(osis)
|
|
if (!match) return osis
|
|
return `${osisBookToUsx(match[1])}.${Number(match[2])}.${Number(match[3])}`
|
|
}
|
|
|
|
function osisBookToUsx(book) {
|
|
const map = {
|
|
Gen: 'GEN',
|
|
Exod: 'EXO',
|
|
Lev: 'LEV',
|
|
Num: 'NUM',
|
|
Deut: 'DEU',
|
|
Josh: 'JOS',
|
|
Judg: 'JDG',
|
|
Ruth: 'RUT',
|
|
'1Sam': '1SA',
|
|
'2Sam': '2SA',
|
|
'1Kgs': '1KI',
|
|
'2Kgs': '2KI',
|
|
'1Chr': '1CH',
|
|
'2Chr': '2CH',
|
|
Ezra: 'EZR',
|
|
Neh: 'NEH',
|
|
Esth: 'EST',
|
|
Job: 'JOB',
|
|
Ps: 'PSA',
|
|
Prov: 'PRO',
|
|
Eccl: 'ECC',
|
|
Song: 'SNG',
|
|
Isa: 'ISA',
|
|
Jer: 'JER',
|
|
Lam: 'LAM',
|
|
Ezek: 'EZK',
|
|
Dan: 'DAN',
|
|
Hos: 'HOS',
|
|
Joel: 'JOL',
|
|
Amos: 'AMO',
|
|
Obad: 'OBA',
|
|
Jonah: 'JON',
|
|
Mic: 'MIC',
|
|
Nah: 'NAM',
|
|
Hab: 'HAB',
|
|
Zeph: 'ZEP',
|
|
Hag: 'HAG',
|
|
Zech: 'ZEC',
|
|
Mal: 'MAL',
|
|
Matt: 'MAT',
|
|
Mark: 'MRK',
|
|
Luke: 'LUK',
|
|
John: 'JHN',
|
|
Acts: 'ACT',
|
|
Rom: 'ROM',
|
|
'1Cor': '1CO',
|
|
'2Cor': '2CO',
|
|
Gal: 'GAL',
|
|
Eph: 'EPH',
|
|
Phil: 'PHP',
|
|
Col: 'COL',
|
|
'1Thess': '1TH',
|
|
'2Thess': '2TH',
|
|
'1Tim': '1TI',
|
|
'2Tim': '2TI',
|
|
Titus: 'TIT',
|
|
Phlm: 'PHM',
|
|
Heb: 'HEB',
|
|
Jas: 'JAS',
|
|
'1Pet': '1PE',
|
|
'2Pet': '2PE',
|
|
'1John': '1JN',
|
|
'2John': '2JN',
|
|
'3John': '3JN',
|
|
Jude: 'JUD',
|
|
Rev: 'REV',
|
|
}
|
|
return map[book] ?? book.toUpperCase()
|
|
}
|
|
|
|
async function writeJsonl(filePath, rows) {
|
|
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
|
}
|
|
|
|
async function fileInfo(filePath) {
|
|
const data = await readFile(filePath)
|
|
return {
|
|
path: path.basename(filePath),
|
|
bytes: data.length,
|
|
sha256: await sha256File(filePath),
|
|
}
|
|
}
|