import AdmZip from 'adm-zip' import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' import path from 'node:path' import { downloadFile, readJson, sha256File, writeJson } from './lib.js' const manifestPath = process.argv[2] if (!manifestPath) { console.error('Usage: node scripts/import-openbible-places.js sources/.json') process.exit(1) } const root = process.cwd() const manifest = await readJson(path.resolve(root, manifestPath)) const zipPath = path.join(root, 'cache', manifest.id, `${manifest.id}.zip`) const extractDir = path.join(root, 'cache', manifest.id, 'source') const packageDir = path.join(root, 'packages', 'json', manifest.id) await downloadFile(manifest.source.download_url, zipPath) const sourceSha = await sha256File(zipPath) if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) { throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`) } await rm(extractDir, { recursive: true, force: true }) await mkdir(extractDir, { recursive: true }) new AdmZip(zipPath).extractAllTo(extractDir, true) const sourceRoot = await findExtractedRoot(extractDir, 'data/ancient.jsonl') const ancientRows = await readJsonl(path.join(sourceRoot, 'data', 'ancient.jsonl')) const modernRows = await readJsonl(path.join(sourceRoot, 'data', 'modern.jsonl')) const modernLocations = modernRows.map((row) => ({ id: row.id, name: row.friendly_id, type: row.type ?? row.class ?? '', class: row.class ?? '', url_slug: row.url_slug ?? '', geometry: row.geometry ?? '', land_or_water: row.land_or_water ?? '', lon: parseLonLat(row.lonlat).lon, lat: parseLonLat(row.lonlat).lat, lonlat_type: row.lonlat_type ?? row.precision?.description ?? '', precision: row.precision ?? null, names: row.names ?? [], media: normalizeMedia(row.media), source: { geometry_credit: row.geometry_credit ?? '', coordinates_source: row.coordinates_source ?? null, secondary_sources: row.secondary_sources ?? [], }, })) const modernById = new Map(modernLocations.map((location) => [location.id, location])) const places = [] const versePlaces = [] const placeImages = [] for (const row of ancientRows) { const best = bestIdentification(row, modernById) const place = { id: row.id, name: row.friendly_id, url_slug: row.url_slug ?? '', types: row.types ?? [], preceding_article: row.preceding_article ?? '', comment: row.comment ?? '', translation_name_counts: row.translation_name_counts ?? {}, linked_data: row.linked_data ?? {}, geometry_credit: row.geometry_credit ?? '', geojson_file: row.geojson_file ?? '', best_modern_location_id: best?.id ?? '', best_modern_location_name: best?.name ?? '', lon: best?.lon ?? null, lat: best?.lat ?? null, confidence: best?.confidence ?? null, confidence_label: confidenceLabel(best?.confidence), media: normalizeMedia(row.media), identifications: (row.identifications ?? []).map((identification) => ({ id: identification.id, description: stripTags(identification.description ?? ''), class: identification.class ?? '', types: identification.resolutions?.flatMap((resolution) => resolution.types ?? []) ?? [], score: identification.score?.time_total ?? identification.score?.vote_total ?? null, })), } places.push(place) const thumbnail = place.media?.thumbnail if (thumbnail?.file || thumbnail?.credit_url) { placeImages.push({ id: `${place.id}:thumbnail`, place_id: place.id, modern_location_id: place.best_modern_location_id, kind: 'thumbnail', file: thumbnail.file ?? '', image_id: thumbnail.image_id ?? '', description: stripTags(thumbnail.description ?? ''), credit: thumbnail.credit ?? '', credit_url: thumbnail.credit_url ?? '', placeholder: thumbnail.placeholder ?? '', }) } for (const verse of row.verses ?? []) { versePlaces.push({ id: `${osisToLibreVerseId(verse.osis)}:${row.id}`, verse_id: osisToLibreVerseId(verse.osis), osis: verse.osis, usx: verse.usx ?? '', readable: verse.readable ?? '', translations: verse.translations ?? [], place_id: row.id, place_name: row.friendly_id, place_types: row.types ?? [], modern_location_id: place.best_modern_location_id, modern_location_name: place.best_modern_location_name, lon: place.lon, lat: place.lat, confidence: place.confidence, confidence_label: place.confidence_label, instance_types: verse.instance_types ?? {}, }) } } places.sort((a, b) => a.name.localeCompare(b.name)) modernLocations.sort((a, b) => a.name.localeCompare(b.name)) versePlaces.sort((a, b) => a.verse_id.localeCompare(b.verse_id) || a.place_name.localeCompare(b.place_name)) placeImages.sort((a, b) => a.place_id.localeCompare(b.place_id)) await mkdir(packageDir, { recursive: true }) await writeJsonl(path.join(packageDir, 'places.jsonl'), places) await writeJsonl(path.join(packageDir, 'modern-locations.jsonl'), modernLocations) await writeJsonl(path.join(packageDir, 'verse-places.jsonl'), versePlaces) await writeJsonl(path.join(packageDir, 'place-images.jsonl'), placeImages) const files = { places_jsonl: await fileInfo(path.join(packageDir, 'places.jsonl')), modern_locations_jsonl: await fileInfo(path.join(packageDir, 'modern-locations.jsonl')), verse_places_jsonl: await fileInfo(path.join(packageDir, 'verse-places.jsonl')), place_images_jsonl: await fileInfo(path.join(packageDir, 'place-images.jsonl')), } await writeJson(path.join(packageDir, 'catalog.json'), { schema_version: 'librebible.resource-catalog.v1', project: manifest.project_name ?? 'LibreBible', id: manifest.id, resource_type: manifest.resource_type, title: manifest.title, short_title: manifest.short_title, abbreviation: manifest.abbreviation, alternate_ids: manifest.alternate_ids ?? [], language: manifest.language, script: manifest.script, canon: manifest.canon, contributors: manifest.contributors ?? [], features: manifest.features ?? [], attachments: manifest.attachments ?? {}, source: manifest.source, license: manifest.license, catalog_display: manifest.catalog_display, importer: manifest.importer, source_manifest_file: path.basename(manifestPath), generated_at: new Date().toISOString(), source_sha256: sourceSha, checks: manifest.checks, counts: { entries: places.length, places: places.length, modern_locations: modernLocations.length, verse_place_links: versePlaces.length, verses_with_places: new Set(versePlaces.map((entry) => entry.verse_id)).size, place_images: placeImages.length, }, files, }) console.log(`${manifest.id}: imported ${places.length} places, ${modernLocations.length} modern locations, ${versePlaces.length} verse-place links`) async function findExtractedRoot(baseDir, requiredRelativePath) { const entries = await import('node:fs/promises').then((fs) => fs.readdir(baseDir, { withFileTypes: true })) for (const entry of entries) { if (!entry.isDirectory()) continue const candidate = path.join(baseDir, entry.name) try { await readFile(path.join(candidate, requiredRelativePath), 'utf8') return candidate } catch { // Keep looking. } } throw new Error(`Unable to find extracted source containing ${requiredRelativePath}`) } async function readJsonl(filePath) { const text = await readFile(filePath, 'utf8') return text .split('\n') .filter((line) => line.trim().length > 0) .map((line) => JSON.parse(line)) } function bestIdentification(row, modernById) { const associations = Object.entries(row.modern_associations ?? {}) .map(([id, value]) => ({ id, name: value.name, confidence: Number(value.score ?? 0), })) .sort((left, right) => right.confidence - left.confidence) for (const association of associations) { const modern = modernById.get(association.id) if (!modern) continue return { ...modern, confidence: association.confidence, } } for (const identification of row.identifications ?? []) { const modern = modernById.get(identification.id) if (!modern) continue return { ...modern, confidence: identification.score?.time_total ?? identification.score?.vote_total ?? null, } } return null } function confidenceLabel(score) { if (score == null) return 'unknown' if (score >= 800) return 'high' if (score >= 400) return 'medium' if (score > 0) return 'low' return 'unknown' } function normalizeMedia(media) { if (!media) return null return { thumbnail: media.thumbnail ? { file: media.thumbnail.file ?? '', image_id: media.thumbnail.image_id ?? '', description: stripTags(media.thumbnail.description ?? ''), credit: media.thumbnail.credit ?? '', credit_url: media.thumbnail.credit_url ?? '', placeholder: media.thumbnail.placeholder ?? '', } : null, } } function parseLonLat(value) { const [lon, lat] = String(value ?? '').split(',').map((part) => Number(part.trim())) return { lon: Number.isFinite(lon) ? lon : null, lat: Number.isFinite(lat) ? lat : null, } } function stripTags(value) { return String(value).replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim() } function osisToLibreVerseId(osis) { const match = /^([1-3]?[A-Za-z]+)\.(\d+)\.(\d+)$/.exec(osis) if (!match) return osis return `${osisBookToUsx(match[1])}.${Number(match[2])}.${Number(match[3])}` } function osisBookToUsx(book) { const map = { Gen: 'GEN', Exod: 'EXO', Lev: 'LEV', Num: 'NUM', Deut: 'DEU', Josh: 'JOS', Judg: 'JDG', Ruth: 'RUT', '1Sam': '1SA', '2Sam': '2SA', '1Kgs': '1KI', '2Kgs': '2KI', '1Chr': '1CH', '2Chr': '2CH', Ezra: 'EZR', Neh: 'NEH', Esth: 'EST', Job: 'JOB', Ps: 'PSA', Prov: 'PRO', Eccl: 'ECC', Song: 'SNG', Isa: 'ISA', Jer: 'JER', Lam: 'LAM', Ezek: 'EZK', Dan: 'DAN', Hos: 'HOS', Joel: 'JOL', Amos: 'AMO', Obad: 'OBA', Jonah: 'JON', Mic: 'MIC', Nah: 'NAM', Hab: 'HAB', Zeph: 'ZEP', Hag: 'HAG', Zech: 'ZEC', Mal: 'MAL', Matt: 'MAT', Mark: 'MRK', Luke: 'LUK', John: 'JHN', Acts: 'ACT', Rom: 'ROM', '1Cor': '1CO', '2Cor': '2CO', Gal: 'GAL', Eph: 'EPH', Phil: 'PHP', Col: 'COL', '1Thess': '1TH', '2Thess': '2TH', '1Tim': '1TI', '2Tim': '2TI', Titus: 'TIT', Phlm: 'PHM', Heb: 'HEB', Jas: 'JAS', '1Pet': '1PE', '2Pet': '2PE', '1John': '1JN', '2John': '2JN', '3John': '3JN', Jude: 'JUD', Rev: 'REV', } return map[book] ?? book.toUpperCase() } async function writeJsonl(filePath, rows) { await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8') } async function fileInfo(filePath) { const data = await readFile(filePath) return { path: path.basename(filePath), bytes: data.length, sha256: await sha256File(filePath), } }