Add map and KJV audio resource packages

2026-07-15 11:33:32 -05:00
parent b376b07c38
commit 7759c0de6a
210 changed files with 47310 additions and 7 deletions
@@ -0,0 +1,358 @@
+import AdmZip from 'adm-zip'
+import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
+
+const manifestPath = process.argv[2]
+if (!manifestPath) {
+  console.error('Usage: node scripts/import-openbible-places.js sources/<manifest>.json')
+  process.exit(1)
+}
+
+const root = process.cwd()
+const manifest = await readJson(path.resolve(root, manifestPath))
+const zipPath = path.join(root, 'cache', manifest.id, `${manifest.id}.zip`)
+const extractDir = path.join(root, 'cache', manifest.id, 'source')
+const packageDir = path.join(root, 'packages', 'json', manifest.id)
+
+await downloadFile(manifest.source.download_url, zipPath)
+const sourceSha = await sha256File(zipPath)
+if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) {
+  throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`)
+}
+
+await rm(extractDir, { recursive: true, force: true })
+await mkdir(extractDir, { recursive: true })
+new AdmZip(zipPath).extractAllTo(extractDir, true)
+
+const sourceRoot = await findExtractedRoot(extractDir, 'data/ancient.jsonl')
+const ancientRows = await readJsonl(path.join(sourceRoot, 'data', 'ancient.jsonl'))
+const modernRows = await readJsonl(path.join(sourceRoot, 'data', 'modern.jsonl'))
+
+const modernLocations = modernRows.map((row) => ({
+  id: row.id,
+  name: row.friendly_id,
+  type: row.type ?? row.class ?? '',
+  class: row.class ?? '',
+  url_slug: row.url_slug ?? '',
+  geometry: row.geometry ?? '',
+  land_or_water: row.land_or_water ?? '',
+  lon: parseLonLat(row.lonlat).lon,
+  lat: parseLonLat(row.lonlat).lat,
+  lonlat_type: row.lonlat_type ?? row.precision?.description ?? '',
+  precision: row.precision ?? null,
+  names: row.names ?? [],
+  media: normalizeMedia(row.media),
+  source: {
+    geometry_credit: row.geometry_credit ?? '',
+    coordinates_source: row.coordinates_source ?? null,
+    secondary_sources: row.secondary_sources ?? [],
+  },
+}))
+
+const modernById = new Map(modernLocations.map((location) => [location.id, location]))
+const places = []
+const versePlaces = []
+const placeImages = []
+
+for (const row of ancientRows) {
+  const best = bestIdentification(row, modernById)
+  const place = {
+    id: row.id,
+    name: row.friendly_id,
+    url_slug: row.url_slug ?? '',
+    types: row.types ?? [],
+    preceding_article: row.preceding_article ?? '',
+    comment: row.comment ?? '',
+    translation_name_counts: row.translation_name_counts ?? {},
+    linked_data: row.linked_data ?? {},
+    geometry_credit: row.geometry_credit ?? '',
+    geojson_file: row.geojson_file ?? '',
+    best_modern_location_id: best?.id ?? '',
+    best_modern_location_name: best?.name ?? '',
+    lon: best?.lon ?? null,
+    lat: best?.lat ?? null,
+    confidence: best?.confidence ?? null,
+    confidence_label: confidenceLabel(best?.confidence),
+    media: normalizeMedia(row.media),
+    identifications: (row.identifications ?? []).map((identification) => ({
+      id: identification.id,
+      description: stripTags(identification.description ?? ''),
+      class: identification.class ?? '',
+      types: identification.resolutions?.flatMap((resolution) => resolution.types ?? []) ?? [],
+      score: identification.score?.time_total ?? identification.score?.vote_total ?? null,
+    })),
+  }
+  places.push(place)
+
+  const thumbnail = place.media?.thumbnail
+  if (thumbnail?.file || thumbnail?.credit_url) {
+    placeImages.push({
+      id: `${place.id}:thumbnail`,
+      place_id: place.id,
+      modern_location_id: place.best_modern_location_id,
+      kind: 'thumbnail',
+      file: thumbnail.file ?? '',
+      image_id: thumbnail.image_id ?? '',
+      description: stripTags(thumbnail.description ?? ''),
+      credit: thumbnail.credit ?? '',
+      credit_url: thumbnail.credit_url ?? '',
+      placeholder: thumbnail.placeholder ?? '',
+    })
+  }
+
+  for (const verse of row.verses ?? []) {
+    versePlaces.push({
+      id: `${osisToLibreVerseId(verse.osis)}:${row.id}`,
+      verse_id: osisToLibreVerseId(verse.osis),
+      osis: verse.osis,
+      usx: verse.usx ?? '',
+      readable: verse.readable ?? '',
+      translations: verse.translations ?? [],
+      place_id: row.id,
+      place_name: row.friendly_id,
+      place_types: row.types ?? [],
+      modern_location_id: place.best_modern_location_id,
+      modern_location_name: place.best_modern_location_name,
+      lon: place.lon,
+      lat: place.lat,
+      confidence: place.confidence,
+      confidence_label: place.confidence_label,
+      instance_types: verse.instance_types ?? {},
+    })
+  }
+}
+
+places.sort((a, b) => a.name.localeCompare(b.name))
+modernLocations.sort((a, b) => a.name.localeCompare(b.name))
+versePlaces.sort((a, b) => a.verse_id.localeCompare(b.verse_id) || a.place_name.localeCompare(b.place_name))
+placeImages.sort((a, b) => a.place_id.localeCompare(b.place_id))
+
+await mkdir(packageDir, { recursive: true })
+await writeJsonl(path.join(packageDir, 'places.jsonl'), places)
+await writeJsonl(path.join(packageDir, 'modern-locations.jsonl'), modernLocations)
+await writeJsonl(path.join(packageDir, 'verse-places.jsonl'), versePlaces)
+await writeJsonl(path.join(packageDir, 'place-images.jsonl'), placeImages)
+
+const files = {
+  places_jsonl: await fileInfo(path.join(packageDir, 'places.jsonl')),
+  modern_locations_jsonl: await fileInfo(path.join(packageDir, 'modern-locations.jsonl')),
+  verse_places_jsonl: await fileInfo(path.join(packageDir, 'verse-places.jsonl')),
+  place_images_jsonl: await fileInfo(path.join(packageDir, 'place-images.jsonl')),
+}
+
+await writeJson(path.join(packageDir, 'catalog.json'), {
+  schema_version: 'librebible.resource-catalog.v1',
+  project: manifest.project_name ?? 'LibreBible',
+  id: manifest.id,
+  resource_type: manifest.resource_type,
+  title: manifest.title,
+  short_title: manifest.short_title,
+  abbreviation: manifest.abbreviation,
+  alternate_ids: manifest.alternate_ids ?? [],
+  language: manifest.language,
+  script: manifest.script,
+  canon: manifest.canon,
+  contributors: manifest.contributors ?? [],
+  features: manifest.features ?? [],
+  attachments: manifest.attachments ?? {},
+  source: manifest.source,
+  license: manifest.license,
+  catalog_display: manifest.catalog_display,
+  importer: manifest.importer,
+  source_manifest_file: path.basename(manifestPath),
+  generated_at: new Date().toISOString(),
+  source_sha256: sourceSha,
+  checks: manifest.checks,
+  counts: {
+    entries: places.length,
+    places: places.length,
+    modern_locations: modernLocations.length,
+    verse_place_links: versePlaces.length,
+    verses_with_places: new Set(versePlaces.map((entry) => entry.verse_id)).size,
+    place_images: placeImages.length,
+  },
+  files,
+})
+
+console.log(`${manifest.id}: imported ${places.length} places, ${modernLocations.length} modern locations, ${versePlaces.length} verse-place links`)
+
+async function findExtractedRoot(baseDir, requiredRelativePath) {
+  const entries = await import('node:fs/promises').then((fs) => fs.readdir(baseDir, { withFileTypes: true }))
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue
+    const candidate = path.join(baseDir, entry.name)
+    try {
+      await readFile(path.join(candidate, requiredRelativePath), 'utf8')
+      return candidate
+    } catch {
+      // Keep looking.
+    }
+  }
+  throw new Error(`Unable to find extracted source containing ${requiredRelativePath}`)
+}
+
+async function readJsonl(filePath) {
+  const text = await readFile(filePath, 'utf8')
+  return text
+    .split('\n')
+    .filter((line) => line.trim().length > 0)
+    .map((line) => JSON.parse(line))
+}
+
+function bestIdentification(row, modernById) {
+  const associations = Object.entries(row.modern_associations ?? {})
+    .map(([id, value]) => ({
+      id,
+      name: value.name,
+      confidence: Number(value.score ?? 0),
+    }))
+    .sort((left, right) => right.confidence - left.confidence)
+
+  for (const association of associations) {
+    const modern = modernById.get(association.id)
+    if (!modern) continue
+    return {
+      ...modern,
+      confidence: association.confidence,
+    }
+  }
+
+  for (const identification of row.identifications ?? []) {
+    const modern = modernById.get(identification.id)
+    if (!modern) continue
+    return {
+      ...modern,
+      confidence: identification.score?.time_total ?? identification.score?.vote_total ?? null,
+    }
+  }
+
+  return null
+}
+
+function confidenceLabel(score) {
+  if (score == null) return 'unknown'
+  if (score >= 800) return 'high'
+  if (score >= 400) return 'medium'
+  if (score > 0) return 'low'
+  return 'unknown'
+}
+
+function normalizeMedia(media) {
+  if (!media) return null
+  return {
+    thumbnail: media.thumbnail
+      ? {
+          file: media.thumbnail.file ?? '',
+          image_id: media.thumbnail.image_id ?? '',
+          description: stripTags(media.thumbnail.description ?? ''),
+          credit: media.thumbnail.credit ?? '',
+          credit_url: media.thumbnail.credit_url ?? '',
+          placeholder: media.thumbnail.placeholder ?? '',
+        }
+      : null,
+  }
+}
+
+function parseLonLat(value) {
+  const [lon, lat] = String(value ?? '').split(',').map((part) => Number(part.trim()))
+  return {
+    lon: Number.isFinite(lon) ? lon : null,
+    lat: Number.isFinite(lat) ? lat : null,
+  }
+}
+
+function stripTags(value) {
+  return String(value).replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim()
+}
+
+function osisToLibreVerseId(osis) {
+  const match = /^([1-3]?[A-Za-z]+)\.(\d+)\.(\d+)$/.exec(osis)
+  if (!match) return osis
+  return `${osisBookToUsx(match[1])}.${Number(match[2])}.${Number(match[3])}`
+}
+
+function osisBookToUsx(book) {
+  const map = {
+    Gen: 'GEN',
+    Exod: 'EXO',
+    Lev: 'LEV',
+    Num: 'NUM',
+    Deut: 'DEU',
+    Josh: 'JOS',
+    Judg: 'JDG',
+    Ruth: 'RUT',
+    '1Sam': '1SA',
+    '2Sam': '2SA',
+    '1Kgs': '1KI',
+    '2Kgs': '2KI',
+    '1Chr': '1CH',
+    '2Chr': '2CH',
+    Ezra: 'EZR',
+    Neh: 'NEH',
+    Esth: 'EST',
+    Job: 'JOB',
+    Ps: 'PSA',
+    Prov: 'PRO',
+    Eccl: 'ECC',
+    Song: 'SNG',
+    Isa: 'ISA',
+    Jer: 'JER',
+    Lam: 'LAM',
+    Ezek: 'EZK',
+    Dan: 'DAN',
+    Hos: 'HOS',
+    Joel: 'JOL',
+    Amos: 'AMO',
+    Obad: 'OBA',
+    Jonah: 'JON',
+    Mic: 'MIC',
+    Nah: 'NAM',
+    Hab: 'HAB',
+    Zeph: 'ZEP',
+    Hag: 'HAG',
+    Zech: 'ZEC',
+    Mal: 'MAL',
+    Matt: 'MAT',
+    Mark: 'MRK',
+    Luke: 'LUK',
+    John: 'JHN',
+    Acts: 'ACT',
+    Rom: 'ROM',
+    '1Cor': '1CO',
+    '2Cor': '2CO',
+    Gal: 'GAL',
+    Eph: 'EPH',
+    Phil: 'PHP',
+    Col: 'COL',
+    '1Thess': '1TH',
+    '2Thess': '2TH',
+    '1Tim': '1TI',
+    '2Tim': '2TI',
+    Titus: 'TIT',
+    Phlm: 'PHM',
+    Heb: 'HEB',
+    Jas: 'JAS',
+    '1Pet': '1PE',
+    '2Pet': '2PE',
+    '1John': '1JN',
+    '2John': '2JN',
+    '3John': '3JN',
+    Jude: 'JUD',
+    Rev: 'REV',
+  }
+  return map[book] ?? book.toUpperCase()
+}
+
+async function writeJsonl(filePath, rows) {
+  await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
+}
+
+async function fileInfo(filePath) {
+  const data = await readFile(filePath)
+  return {
+    path: path.basename(filePath),
+    bytes: data.length,
+    sha256: await sha256File(filePath),
+  }
+}