Add map and KJV audio resource packages
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
import AdmZip from 'adm-zip'
|
||||
import { copyFile, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
|
||||
|
||||
const manifestPath = process.argv[2]
|
||||
if (!manifestPath) {
|
||||
console.error('Usage: node scripts/import-ubs-bible-routes.js sources/<manifest>.json')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const root = process.cwd()
|
||||
const manifest = await readJson(path.resolve(root, manifestPath))
|
||||
const zipPath = path.join(root, 'cache', manifest.id, `${manifest.id}.zip`)
|
||||
const extractDir = path.join(root, 'cache', manifest.id, 'source')
|
||||
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
||||
const geojsonOutDir = path.join(packageDir, 'geojson')
|
||||
|
||||
await downloadFile(manifest.source.download_url, zipPath)
|
||||
const sourceSha = await sha256File(zipPath)
|
||||
if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
||||
throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`)
|
||||
}
|
||||
|
||||
await rm(extractDir, { recursive: true, force: true })
|
||||
await rm(packageDir, { recursive: true, force: true })
|
||||
await mkdir(extractDir, { recursive: true })
|
||||
await mkdir(geojsonOutDir, { recursive: true })
|
||||
new AdmZip(zipPath).extractAllTo(extractDir, true)
|
||||
|
||||
const sourceRoot = await findExtractedRoot(extractDir, 'ubs-bible-routes/metadata.csv')
|
||||
const routeRoot = path.join(sourceRoot, 'ubs-bible-routes')
|
||||
const geojsonSourceDir = path.join(routeRoot, 'GeoJsonRoutes')
|
||||
const metadata = parseMetadata(await readFile(path.join(routeRoot, 'metadata.csv'), 'utf8'))
|
||||
const metadataByTitle = new Map(metadata.map((row) => [normalizeTitle(row.image_file), row]))
|
||||
const geojsonFiles = (await readdir(geojsonSourceDir))
|
||||
.filter((file) => file.toLowerCase().endsWith('.geojson'))
|
||||
.sort((a, b) => a.localeCompare(b))
|
||||
|
||||
const routes = []
|
||||
|
||||
for (const file of geojsonFiles) {
|
||||
const idMatch = /^(\d+)/.exec(file)
|
||||
const routeNumber = idMatch ? Number(idMatch[1]) : routes.length + 1
|
||||
const title = routeTitleFromFilename(file)
|
||||
const chapter = metadataByTitle.get(normalizeTitle(file))
|
||||
const outputName = `${String(routeNumber).padStart(3, '0')}-${slug(title)}.geojson`
|
||||
const sourceFile = path.join(geojsonSourceDir, file)
|
||||
const outputFile = path.join(geojsonOutDir, outputName)
|
||||
await copyFile(sourceFile, outputFile)
|
||||
|
||||
const geojson = JSON.parse(await readFile(sourceFile, 'utf8'))
|
||||
routes.push({
|
||||
id: `ubs-route-${String(routeNumber).padStart(3, '0')}`,
|
||||
route_number: routeNumber,
|
||||
title,
|
||||
section_number: chapter?.section_number ?? '',
|
||||
section_title: chapter?.section_title ?? '',
|
||||
map_number: chapter?.map_number ?? '',
|
||||
source_file: `GeoJsonRoutes/${file}`,
|
||||
geojson_file: `geojson/${outputName}`,
|
||||
geometry_types: geometryTypes(geojson),
|
||||
bounds: boundsForGeojson(geojson),
|
||||
})
|
||||
}
|
||||
|
||||
routes.sort((a, b) => a.route_number - b.route_number)
|
||||
await writeJsonl(path.join(packageDir, 'routes.jsonl'), routes)
|
||||
|
||||
const files = {
|
||||
routes_jsonl: await fileInfo(path.join(packageDir, 'routes.jsonl')),
|
||||
geojson_dir: {
|
||||
path: 'geojson',
|
||||
files: geojsonFiles.length,
|
||||
},
|
||||
}
|
||||
|
||||
await writeJson(path.join(packageDir, 'catalog.json'), {
|
||||
schema_version: 'librebible.resource-catalog.v1',
|
||||
project: manifest.project_name ?? 'LibreBible',
|
||||
id: manifest.id,
|
||||
resource_type: manifest.resource_type,
|
||||
title: manifest.title,
|
||||
short_title: manifest.short_title,
|
||||
abbreviation: manifest.abbreviation,
|
||||
alternate_ids: manifest.alternate_ids ?? [],
|
||||
language: manifest.language,
|
||||
script: manifest.script,
|
||||
canon: manifest.canon,
|
||||
contributors: manifest.contributors ?? [],
|
||||
features: manifest.features ?? [],
|
||||
attachments: manifest.attachments ?? {},
|
||||
source: manifest.source,
|
||||
license: manifest.license,
|
||||
catalog_display: manifest.catalog_display,
|
||||
importer: manifest.importer,
|
||||
source_manifest_file: path.basename(manifestPath),
|
||||
generated_at: new Date().toISOString(),
|
||||
source_sha256: sourceSha,
|
||||
checks: manifest.checks,
|
||||
counts: {
|
||||
entries: routes.length,
|
||||
routes: routes.length,
|
||||
geojson_files: geojsonFiles.length,
|
||||
sections: new Set(routes.map((route) => route.section_number).filter(Boolean)).size,
|
||||
},
|
||||
files,
|
||||
})
|
||||
|
||||
console.log(`${manifest.id}: imported ${routes.length} routes`)
|
||||
|
||||
async function findExtractedRoot(baseDir, requiredRelativePath) {
|
||||
const entries = await readdir(baseDir, { withFileTypes: true })
|
||||
for (const entry of entries) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const candidate = path.join(baseDir, entry.name)
|
||||
try {
|
||||
await readFile(path.join(candidate, requiredRelativePath), 'utf8')
|
||||
return candidate
|
||||
} catch {
|
||||
// Keep looking.
|
||||
}
|
||||
}
|
||||
throw new Error(`Unable to find extracted source containing ${requiredRelativePath}`)
|
||||
}
|
||||
|
||||
function parseMetadata(text) {
|
||||
return text
|
||||
.replace(/\r\n/g, '\n')
|
||||
.split('\n')
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => {
|
||||
const columns = line.split('\t')
|
||||
return {
|
||||
section_number: columns[0]?.trim() ?? '',
|
||||
section_title: columns[1]?.trim() ?? '',
|
||||
map_number: columns[2]?.trim() ?? '',
|
||||
image_file: columns[3]?.trim() ?? '',
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function routeTitleFromFilename(file) {
|
||||
return file.replace(/^\d+\.\s*/, '').replace(/\.geojson$/i, '').trim()
|
||||
}
|
||||
|
||||
function normalizeTitle(file) {
|
||||
return String(file)
|
||||
.replace(/^\d+\.\s*/, '')
|
||||
.replace(/\.(geojson|jpg|jpeg|png|svg)$/i, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
}
|
||||
|
||||
function geometryTypes(geojson) {
|
||||
const types = new Set()
|
||||
visitCoordinates(geojson, (_lon, _lat, type) => {
|
||||
if (type) types.add(type)
|
||||
})
|
||||
return [...types].sort()
|
||||
}
|
||||
|
||||
function boundsForGeojson(geojson) {
|
||||
const bounds = {
|
||||
min_lon: Infinity,
|
||||
min_lat: Infinity,
|
||||
max_lon: -Infinity,
|
||||
max_lat: -Infinity,
|
||||
}
|
||||
visitCoordinates(geojson, (lon, lat) => {
|
||||
if (!Number.isFinite(lon) || !Number.isFinite(lat)) return
|
||||
bounds.min_lon = Math.min(bounds.min_lon, lon)
|
||||
bounds.min_lat = Math.min(bounds.min_lat, lat)
|
||||
bounds.max_lon = Math.max(bounds.max_lon, lon)
|
||||
bounds.max_lat = Math.max(bounds.max_lat, lat)
|
||||
})
|
||||
if (!Number.isFinite(bounds.min_lon)) return null
|
||||
return bounds
|
||||
}
|
||||
|
||||
function visitCoordinates(geojson, callback) {
|
||||
if (geojson?.type === 'FeatureCollection') {
|
||||
for (const feature of geojson.features ?? []) visitCoordinates(feature, callback)
|
||||
return
|
||||
}
|
||||
if (geojson?.type === 'Feature') {
|
||||
visitGeometry(geojson.geometry, callback)
|
||||
return
|
||||
}
|
||||
visitGeometry(geojson, callback)
|
||||
}
|
||||
|
||||
function visitGeometry(geometry, callback) {
|
||||
if (!geometry) return
|
||||
const type = geometry.type
|
||||
if (type === 'GeometryCollection') {
|
||||
for (const child of geometry.geometries ?? []) visitGeometry(child, callback)
|
||||
return
|
||||
}
|
||||
walkCoordinates(geometry.coordinates, type, callback)
|
||||
}
|
||||
|
||||
function walkCoordinates(value, type, callback) {
|
||||
if (!Array.isArray(value)) return
|
||||
if (typeof value[0] === 'number' && typeof value[1] === 'number') {
|
||||
callback(value[0], value[1], type)
|
||||
return
|
||||
}
|
||||
for (const child of value) walkCoordinates(child, type, callback)
|
||||
}
|
||||
|
||||
function slug(value) {
|
||||
return String(value).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
||||
}
|
||||
|
||||
async function writeJsonl(filePath, rows) {
|
||||
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
||||
}
|
||||
|
||||
async function fileInfo(filePath) {
|
||||
const data = await readFile(filePath)
|
||||
return {
|
||||
path: path.basename(filePath),
|
||||
bytes: data.length,
|
||||
sha256: await sha256File(filePath),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user