229 lines
7.4 KiB
JavaScript
229 lines
7.4 KiB
JavaScript
import AdmZip from 'adm-zip'
|
|
import { copyFile, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import { downloadFile, readJson, sha256File, writeJson } from './lib.js'
|
|
|
|
const manifestPath = process.argv[2]
|
|
if (!manifestPath) {
|
|
console.error('Usage: node scripts/import-ubs-bible-routes.js sources/<manifest>.json')
|
|
process.exit(1)
|
|
}
|
|
|
|
const root = process.cwd()
|
|
const manifest = await readJson(path.resolve(root, manifestPath))
|
|
const zipPath = path.join(root, 'cache', manifest.id, `${manifest.id}.zip`)
|
|
const extractDir = path.join(root, 'cache', manifest.id, 'source')
|
|
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
|
const geojsonOutDir = path.join(packageDir, 'geojson')
|
|
|
|
await downloadFile(manifest.source.download_url, zipPath)
|
|
const sourceSha = await sha256File(zipPath)
|
|
if (sourceSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
|
throw new Error(`${manifest.id}: source checksum mismatch. Expected ${manifest.checks.expected_sha256}, got ${sourceSha}`)
|
|
}
|
|
|
|
await rm(extractDir, { recursive: true, force: true })
|
|
await rm(packageDir, { recursive: true, force: true })
|
|
await mkdir(extractDir, { recursive: true })
|
|
await mkdir(geojsonOutDir, { recursive: true })
|
|
new AdmZip(zipPath).extractAllTo(extractDir, true)
|
|
|
|
const sourceRoot = await findExtractedRoot(extractDir, 'ubs-bible-routes/metadata.csv')
|
|
const routeRoot = path.join(sourceRoot, 'ubs-bible-routes')
|
|
const geojsonSourceDir = path.join(routeRoot, 'GeoJsonRoutes')
|
|
const metadata = parseMetadata(await readFile(path.join(routeRoot, 'metadata.csv'), 'utf8'))
|
|
const metadataByTitle = new Map(metadata.map((row) => [normalizeTitle(row.image_file), row]))
|
|
const geojsonFiles = (await readdir(geojsonSourceDir))
|
|
.filter((file) => file.toLowerCase().endsWith('.geojson'))
|
|
.sort((a, b) => a.localeCompare(b))
|
|
|
|
const routes = []
|
|
|
|
for (const file of geojsonFiles) {
|
|
const idMatch = /^(\d+)/.exec(file)
|
|
const routeNumber = idMatch ? Number(idMatch[1]) : routes.length + 1
|
|
const title = routeTitleFromFilename(file)
|
|
const chapter = metadataByTitle.get(normalizeTitle(file))
|
|
const outputName = `${String(routeNumber).padStart(3, '0')}-${slug(title)}.geojson`
|
|
const sourceFile = path.join(geojsonSourceDir, file)
|
|
const outputFile = path.join(geojsonOutDir, outputName)
|
|
await copyFile(sourceFile, outputFile)
|
|
|
|
const geojson = JSON.parse(await readFile(sourceFile, 'utf8'))
|
|
routes.push({
|
|
id: `ubs-route-${String(routeNumber).padStart(3, '0')}`,
|
|
route_number: routeNumber,
|
|
title,
|
|
section_number: chapter?.section_number ?? '',
|
|
section_title: chapter?.section_title ?? '',
|
|
map_number: chapter?.map_number ?? '',
|
|
source_file: `GeoJsonRoutes/${file}`,
|
|
geojson_file: `geojson/${outputName}`,
|
|
geometry_types: geometryTypes(geojson),
|
|
bounds: boundsForGeojson(geojson),
|
|
})
|
|
}
|
|
|
|
routes.sort((a, b) => a.route_number - b.route_number)
|
|
await writeJsonl(path.join(packageDir, 'routes.jsonl'), routes)
|
|
|
|
const files = {
|
|
routes_jsonl: await fileInfo(path.join(packageDir, 'routes.jsonl')),
|
|
geojson_dir: {
|
|
path: 'geojson',
|
|
files: geojsonFiles.length,
|
|
},
|
|
}
|
|
|
|
await writeJson(path.join(packageDir, 'catalog.json'), {
|
|
schema_version: 'librebible.resource-catalog.v1',
|
|
project: manifest.project_name ?? 'LibreBible',
|
|
id: manifest.id,
|
|
resource_type: manifest.resource_type,
|
|
title: manifest.title,
|
|
short_title: manifest.short_title,
|
|
abbreviation: manifest.abbreviation,
|
|
alternate_ids: manifest.alternate_ids ?? [],
|
|
language: manifest.language,
|
|
script: manifest.script,
|
|
canon: manifest.canon,
|
|
contributors: manifest.contributors ?? [],
|
|
features: manifest.features ?? [],
|
|
attachments: manifest.attachments ?? {},
|
|
source: manifest.source,
|
|
license: manifest.license,
|
|
catalog_display: manifest.catalog_display,
|
|
importer: manifest.importer,
|
|
source_manifest_file: path.basename(manifestPath),
|
|
generated_at: new Date().toISOString(),
|
|
source_sha256: sourceSha,
|
|
checks: manifest.checks,
|
|
counts: {
|
|
entries: routes.length,
|
|
routes: routes.length,
|
|
geojson_files: geojsonFiles.length,
|
|
sections: new Set(routes.map((route) => route.section_number).filter(Boolean)).size,
|
|
},
|
|
files,
|
|
})
|
|
|
|
console.log(`${manifest.id}: imported ${routes.length} routes`)
|
|
|
|
async function findExtractedRoot(baseDir, requiredRelativePath) {
|
|
const entries = await readdir(baseDir, { withFileTypes: true })
|
|
for (const entry of entries) {
|
|
if (!entry.isDirectory()) continue
|
|
const candidate = path.join(baseDir, entry.name)
|
|
try {
|
|
await readFile(path.join(candidate, requiredRelativePath), 'utf8')
|
|
return candidate
|
|
} catch {
|
|
// Keep looking.
|
|
}
|
|
}
|
|
throw new Error(`Unable to find extracted source containing ${requiredRelativePath}`)
|
|
}
|
|
|
|
function parseMetadata(text) {
|
|
return text
|
|
.replace(/\r\n/g, '\n')
|
|
.split('\n')
|
|
.filter((line) => line.trim().length > 0)
|
|
.map((line) => {
|
|
const columns = line.split('\t')
|
|
return {
|
|
section_number: columns[0]?.trim() ?? '',
|
|
section_title: columns[1]?.trim() ?? '',
|
|
map_number: columns[2]?.trim() ?? '',
|
|
image_file: columns[3]?.trim() ?? '',
|
|
}
|
|
})
|
|
}
|
|
|
|
function routeTitleFromFilename(file) {
|
|
return file.replace(/^\d+\.\s*/, '').replace(/\.geojson$/i, '').trim()
|
|
}
|
|
|
|
function normalizeTitle(file) {
|
|
return String(file)
|
|
.replace(/^\d+\.\s*/, '')
|
|
.replace(/\.(geojson|jpg|jpeg|png|svg)$/i, '')
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
.toLowerCase()
|
|
}
|
|
|
|
function geometryTypes(geojson) {
|
|
const types = new Set()
|
|
visitCoordinates(geojson, (_lon, _lat, type) => {
|
|
if (type) types.add(type)
|
|
})
|
|
return [...types].sort()
|
|
}
|
|
|
|
function boundsForGeojson(geojson) {
|
|
const bounds = {
|
|
min_lon: Infinity,
|
|
min_lat: Infinity,
|
|
max_lon: -Infinity,
|
|
max_lat: -Infinity,
|
|
}
|
|
visitCoordinates(geojson, (lon, lat) => {
|
|
if (!Number.isFinite(lon) || !Number.isFinite(lat)) return
|
|
bounds.min_lon = Math.min(bounds.min_lon, lon)
|
|
bounds.min_lat = Math.min(bounds.min_lat, lat)
|
|
bounds.max_lon = Math.max(bounds.max_lon, lon)
|
|
bounds.max_lat = Math.max(bounds.max_lat, lat)
|
|
})
|
|
if (!Number.isFinite(bounds.min_lon)) return null
|
|
return bounds
|
|
}
|
|
|
|
function visitCoordinates(geojson, callback) {
|
|
if (geojson?.type === 'FeatureCollection') {
|
|
for (const feature of geojson.features ?? []) visitCoordinates(feature, callback)
|
|
return
|
|
}
|
|
if (geojson?.type === 'Feature') {
|
|
visitGeometry(geojson.geometry, callback)
|
|
return
|
|
}
|
|
visitGeometry(geojson, callback)
|
|
}
|
|
|
|
function visitGeometry(geometry, callback) {
|
|
if (!geometry) return
|
|
const type = geometry.type
|
|
if (type === 'GeometryCollection') {
|
|
for (const child of geometry.geometries ?? []) visitGeometry(child, callback)
|
|
return
|
|
}
|
|
walkCoordinates(geometry.coordinates, type, callback)
|
|
}
|
|
|
|
function walkCoordinates(value, type, callback) {
|
|
if (!Array.isArray(value)) return
|
|
if (typeof value[0] === 'number' && typeof value[1] === 'number') {
|
|
callback(value[0], value[1], type)
|
|
return
|
|
}
|
|
for (const child of value) walkCoordinates(child, type, callback)
|
|
}
|
|
|
|
function slug(value) {
|
|
return String(value).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '')
|
|
}
|
|
|
|
async function writeJsonl(filePath, rows) {
|
|
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
|
}
|
|
|
|
async function fileInfo(filePath) {
|
|
const data = await readFile(filePath)
|
|
return {
|
|
path: path.basename(filePath),
|
|
bytes: data.length,
|
|
sha256: await sha256File(filePath),
|
|
}
|
|
}
|