273 lines
8.3 KiB
JavaScript
273 lines
8.3 KiB
JavaScript
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import { readJson, sha256File, writeJson } from './lib.js'
|
|
|
|
const manifestPath = process.argv[2]
|
|
if (!manifestPath) {
|
|
console.error('Usage: node scripts/import-kjv-audio-eliranwong.js sources/kjv-audio-eliranwong.json')
|
|
process.exit(1)
|
|
}
|
|
|
|
const root = process.cwd()
|
|
const manifest = await readJson(path.resolve(root, manifestPath))
|
|
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
|
const kjvVersesPath = path.join(root, 'packages', 'json', 'kjv-eng-kjv2006', 'verses.jsonl')
|
|
|
|
const bookOrder = [
|
|
['GEN', 'Genesis'],
|
|
['EXO', 'Exodus'],
|
|
['LEV', 'Leviticus'],
|
|
['NUM', 'Numbers'],
|
|
['DEU', 'Deuteronomy'],
|
|
['JOS', 'Joshua'],
|
|
['JDG', 'Judges'],
|
|
['RUT', 'Ruth'],
|
|
['1SA', '1Samuel'],
|
|
['2SA', '2Samuel'],
|
|
['1KI', '1Kings'],
|
|
['2KI', '2Kings'],
|
|
['1CH', '1Chronicles'],
|
|
['2CH', '2Chronicles'],
|
|
['EZR', 'Ezra'],
|
|
['NEH', 'Nehemiah'],
|
|
['EST', 'Esther'],
|
|
['JOB', 'Job'],
|
|
['PSA', 'Psalms'],
|
|
['PRO', 'Proverbs'],
|
|
['ECC', 'Ecclesiastes'],
|
|
['SNG', 'SongOfSolomon'],
|
|
['ISA', 'Isaiah'],
|
|
['JER', 'Jeremiah'],
|
|
['LAM', 'Lamentations'],
|
|
['EZK', 'Ezekiel'],
|
|
['DAN', 'Daniel'],
|
|
['HOS', 'Hosea'],
|
|
['JOL', 'Joel'],
|
|
['AMO', 'Amos'],
|
|
['OBA', 'Obadiah'],
|
|
['JON', 'Jonah'],
|
|
['MIC', 'Micah'],
|
|
['NAM', 'Nahum'],
|
|
['HAB', 'Habakkuk'],
|
|
['ZEP', 'Zephaniah'],
|
|
['HAG', 'Haggai'],
|
|
['ZEC', 'Zechariah'],
|
|
['MAL', 'Malachi'],
|
|
['MAT', 'Matthew'],
|
|
['MRK', 'Mark'],
|
|
['LUK', 'Luke'],
|
|
['JHN', 'John'],
|
|
['ACT', 'Acts'],
|
|
['ROM', 'Romans'],
|
|
['1CO', '1Corinthians'],
|
|
['2CO', '2Corinthians'],
|
|
['GAL', 'Galatians'],
|
|
['EPH', 'Ephesians'],
|
|
['PHP', 'Philippians'],
|
|
['COL', 'Colossians'],
|
|
['1TH', '1Thessalonians'],
|
|
['2TH', '2Thessalonians'],
|
|
['1TI', '1Timothy'],
|
|
['2TI', '2Timothy'],
|
|
['TIT', 'Titus'],
|
|
['PHM', 'Philemon'],
|
|
['HEB', 'Hebrews'],
|
|
['JAS', 'James'],
|
|
['1PE', '1Peter'],
|
|
['2PE', '2Peter'],
|
|
['1JN', '1John'],
|
|
['2JN', '2John'],
|
|
['3JN', '3John'],
|
|
['JUD', 'Jude'],
|
|
['REV', 'Revelation'],
|
|
]
|
|
|
|
const bookById = new Map(
|
|
bookOrder.map(([bookId, audioName], index) => {
|
|
const bookNumber = index + 1
|
|
return [bookId, { bookId, bookNumber, audioName, zipPrefix: String(bookNumber).padStart(2, '0') }]
|
|
}),
|
|
)
|
|
|
|
const verses = (await readFile(kjvVersesPath, 'utf8'))
|
|
.trim()
|
|
.split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line))
|
|
|
|
const audioPackageIndex = await fetchJson(manifest.source.audio_package_index_url)
|
|
const zipEntries = audioPackageIndex
|
|
.filter((entry) => entry.type === 'file' && entry.name.endsWith('.zip'))
|
|
.sort((a, b) => a.name.localeCompare(b.name))
|
|
const zipByPrefix = new Map(zipEntries.map((entry) => [entry.name.slice(0, 2), entry]))
|
|
|
|
const segments = []
|
|
const chapterPlaylists = new Map()
|
|
const bookStats = new Map()
|
|
|
|
for (const verse of verses) {
|
|
const bookInfo = bookById.get(verse.book_id)
|
|
if (!bookInfo) throw new Error(`Unsupported KJV book id: ${verse.book_id}`)
|
|
|
|
const zipEntry = zipByPrefix.get(bookInfo.zipPrefix)
|
|
if (!zipEntry) throw new Error(`No audio ZIP found for ${bookInfo.zipPrefix} ${verse.book}`)
|
|
|
|
const entryPath = `${bookInfo.bookNumber}_${verse.chapter}/KJV_${bookInfo.bookNumber}_${verse.chapter}_${verse.verse}.mp3`
|
|
const segment = {
|
|
id: `${manifest.id}:${verse.id}`,
|
|
resource_id: manifest.id,
|
|
translation_id: 'kjv-eng-kjv2006',
|
|
verse_id: verse.id,
|
|
book_id: verse.book_id,
|
|
book: verse.book,
|
|
book_number: bookInfo.bookNumber,
|
|
chapter: verse.chapter,
|
|
verse: verse.verse,
|
|
reference: verse.reference,
|
|
granularity: 'verse',
|
|
format: 'mp3',
|
|
reader: 'text-to-speech',
|
|
accent: 'British',
|
|
audio_zip_file: zipEntry.name,
|
|
audio_zip_url: zipEntry.download_url,
|
|
audio_zip_bytes: zipEntry.size,
|
|
audio_zip_sha: zipEntry.sha,
|
|
audio_entry_path: entryPath,
|
|
license: manifest.license.name,
|
|
attribution: manifest.license.attribution,
|
|
}
|
|
segments.push(segment)
|
|
|
|
const chapterKey = `${verse.book_id}.${verse.chapter}`
|
|
if (!chapterPlaylists.has(chapterKey)) {
|
|
chapterPlaylists.set(chapterKey, {
|
|
id: `${manifest.id}:${chapterKey}`,
|
|
resource_id: manifest.id,
|
|
translation_id: 'kjv-eng-kjv2006',
|
|
book_id: verse.book_id,
|
|
book: verse.book,
|
|
book_number: bookInfo.bookNumber,
|
|
chapter: verse.chapter,
|
|
reference: `${verse.book} ${verse.chapter}`,
|
|
granularity: 'chapter',
|
|
format: 'mp3-playlist',
|
|
audio_zip_file: zipEntry.name,
|
|
audio_zip_url: zipEntry.download_url,
|
|
audio_zip_bytes: zipEntry.size,
|
|
audio_zip_sha: zipEntry.sha,
|
|
segment_ids: [],
|
|
audio_entry_paths: [],
|
|
})
|
|
}
|
|
const playlist = chapterPlaylists.get(chapterKey)
|
|
playlist.segment_ids.push(segment.id)
|
|
playlist.audio_entry_paths.push(entryPath)
|
|
|
|
if (!bookStats.has(verse.book_id)) {
|
|
bookStats.set(verse.book_id, {
|
|
book_id: verse.book_id,
|
|
book: verse.book,
|
|
book_number: bookInfo.bookNumber,
|
|
chapters: new Set(),
|
|
verses: 0,
|
|
zipEntry,
|
|
})
|
|
}
|
|
const stats = bookStats.get(verse.book_id)
|
|
stats.chapters.add(verse.chapter)
|
|
stats.verses += 1
|
|
}
|
|
|
|
const playlists = Array.from(chapterPlaylists.values()).map((playlist) => ({
|
|
...playlist,
|
|
segment_count: playlist.segment_ids.length,
|
|
}))
|
|
|
|
const bookPackages = Array.from(bookStats.values())
|
|
.sort((a, b) => a.book_number - b.book_number)
|
|
.map((stats) => ({
|
|
id: `${manifest.id}:${String(stats.book_number).padStart(2, '0')}`,
|
|
resource_id: manifest.id,
|
|
translation_id: 'kjv-eng-kjv2006',
|
|
book_id: stats.book_id,
|
|
book: stats.book,
|
|
book_number: stats.book_number,
|
|
chapters: stats.chapters.size,
|
|
verses: stats.verses,
|
|
audio_zip_file: stats.zipEntry.name,
|
|
audio_zip_url: stats.zipEntry.download_url,
|
|
audio_zip_bytes: stats.zipEntry.size,
|
|
audio_zip_sha: stats.zipEntry.sha,
|
|
}))
|
|
|
|
await mkdir(packageDir, { recursive: true })
|
|
await writeJsonl(path.join(packageDir, 'audio-segments.jsonl'), segments)
|
|
await writeJsonl(path.join(packageDir, 'chapter-playlists.jsonl'), playlists)
|
|
await writeJsonl(path.join(packageDir, 'book-packages.jsonl'), bookPackages)
|
|
|
|
const files = {
|
|
audio_segments_jsonl: await fileInfo(path.join(packageDir, 'audio-segments.jsonl'), 'audio-segments.jsonl'),
|
|
chapter_playlists_jsonl: await fileInfo(path.join(packageDir, 'chapter-playlists.jsonl'), 'chapter-playlists.jsonl'),
|
|
book_packages_jsonl: await fileInfo(path.join(packageDir, 'book-packages.jsonl'), 'book-packages.jsonl'),
|
|
}
|
|
|
|
await writeJson(path.join(packageDir, 'catalog.json'), {
|
|
schema_version: 'librebible.resource-catalog.v1',
|
|
project: manifest.project_name,
|
|
id: manifest.id,
|
|
resource_type: manifest.resource_type,
|
|
title: manifest.title,
|
|
short_title: manifest.short_title,
|
|
abbreviation: manifest.abbreviation,
|
|
alternate_ids: manifest.alternate_ids ?? [],
|
|
language: manifest.language,
|
|
script: manifest.script,
|
|
canon: manifest.canon,
|
|
contributors: manifest.contributors ?? [],
|
|
features: manifest.features ?? [],
|
|
attachments: manifest.attachments ?? {},
|
|
source: manifest.source,
|
|
license: manifest.license,
|
|
catalog_display: manifest.catalog_display,
|
|
importer: manifest.importer,
|
|
source_manifest_file: path.basename(manifestPath),
|
|
generated_at: new Date().toISOString(),
|
|
source_sha256: manifest.checks.expected_sha256,
|
|
checks: manifest.checks,
|
|
counts: {
|
|
audio_segments: segments.length,
|
|
chapter_playlists: playlists.length,
|
|
book_packages: bookPackages.length,
|
|
books: bookPackages.length,
|
|
verses: segments.length,
|
|
},
|
|
files,
|
|
})
|
|
|
|
console.log(`Imported ${segments.length} KJV audio segment(s), ${playlists.length} chapter playlist(s), and ${bookPackages.length} book package(s).`)
|
|
|
|
async function fetchJson(url) {
|
|
const response = await fetch(url, {
|
|
headers: {
|
|
Accept: 'application/vnd.github+json',
|
|
'User-Agent': 'LibreBibleDataImporter',
|
|
},
|
|
})
|
|
if (!response.ok) throw new Error(`GitHub API request failed ${response.status} ${response.statusText}: ${url}`)
|
|
return response.json()
|
|
}
|
|
|
|
async function writeJsonl(filePath, rows) {
|
|
await mkdir(path.dirname(filePath), { recursive: true })
|
|
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
|
}
|
|
|
|
async function fileInfo(filePath, relativePath) {
|
|
const stats = await stat(filePath)
|
|
return {
|
|
path: relativePath,
|
|
bytes: stats.size,
|
|
sha256: await sha256File(filePath),
|
|
}
|
|
}
|