import { mkdir, readFile, stat, writeFile } from 'node:fs/promises' import path from 'node:path' import { readJson, sha256File, writeJson } from './lib.js' const manifestPath = process.argv[2] if (!manifestPath) { console.error('Usage: node scripts/import-kjv-audio-eliranwong.js sources/kjv-audio-eliranwong.json') process.exit(1) } const root = process.cwd() const manifest = await readJson(path.resolve(root, manifestPath)) const packageDir = path.join(root, 'packages', 'json', manifest.id) const kjvVersesPath = path.join(root, 'packages', 'json', 'kjv-eng-kjv2006', 'verses.jsonl') const bookOrder = [ ['GEN', 'Genesis'], ['EXO', 'Exodus'], ['LEV', 'Leviticus'], ['NUM', 'Numbers'], ['DEU', 'Deuteronomy'], ['JOS', 'Joshua'], ['JDG', 'Judges'], ['RUT', 'Ruth'], ['1SA', '1Samuel'], ['2SA', '2Samuel'], ['1KI', '1Kings'], ['2KI', '2Kings'], ['1CH', '1Chronicles'], ['2CH', '2Chronicles'], ['EZR', 'Ezra'], ['NEH', 'Nehemiah'], ['EST', 'Esther'], ['JOB', 'Job'], ['PSA', 'Psalms'], ['PRO', 'Proverbs'], ['ECC', 'Ecclesiastes'], ['SNG', 'SongOfSolomon'], ['ISA', 'Isaiah'], ['JER', 'Jeremiah'], ['LAM', 'Lamentations'], ['EZK', 'Ezekiel'], ['DAN', 'Daniel'], ['HOS', 'Hosea'], ['JOL', 'Joel'], ['AMO', 'Amos'], ['OBA', 'Obadiah'], ['JON', 'Jonah'], ['MIC', 'Micah'], ['NAM', 'Nahum'], ['HAB', 'Habakkuk'], ['ZEP', 'Zephaniah'], ['HAG', 'Haggai'], ['ZEC', 'Zechariah'], ['MAL', 'Malachi'], ['MAT', 'Matthew'], ['MRK', 'Mark'], ['LUK', 'Luke'], ['JHN', 'John'], ['ACT', 'Acts'], ['ROM', 'Romans'], ['1CO', '1Corinthians'], ['2CO', '2Corinthians'], ['GAL', 'Galatians'], ['EPH', 'Ephesians'], ['PHP', 'Philippians'], ['COL', 'Colossians'], ['1TH', '1Thessalonians'], ['2TH', '2Thessalonians'], ['1TI', '1Timothy'], ['2TI', '2Timothy'], ['TIT', 'Titus'], ['PHM', 'Philemon'], ['HEB', 'Hebrews'], ['JAS', 'James'], ['1PE', '1Peter'], ['2PE', '2Peter'], ['1JN', '1John'], ['2JN', '2John'], ['3JN', '3John'], ['JUD', 'Jude'], ['REV', 'Revelation'], ] const bookById = new Map( bookOrder.map(([bookId, audioName], index) => { const bookNumber = index + 1 return [bookId, { bookId, bookNumber, audioName, zipPrefix: String(bookNumber).padStart(2, '0') }] }), ) const verses = (await readFile(kjvVersesPath, 'utf8')) .trim() .split(/\r?\n/) .filter(Boolean) .map((line) => JSON.parse(line)) const audioPackageIndex = await fetchJson(manifest.source.audio_package_index_url) const zipEntries = audioPackageIndex .filter((entry) => entry.type === 'file' && entry.name.endsWith('.zip')) .sort((a, b) => a.name.localeCompare(b.name)) const zipByPrefix = new Map(zipEntries.map((entry) => [entry.name.slice(0, 2), entry])) const segments = [] const chapterPlaylists = new Map() const bookStats = new Map() for (const verse of verses) { const bookInfo = bookById.get(verse.book_id) if (!bookInfo) throw new Error(`Unsupported KJV book id: ${verse.book_id}`) const zipEntry = zipByPrefix.get(bookInfo.zipPrefix) if (!zipEntry) throw new Error(`No audio ZIP found for ${bookInfo.zipPrefix} ${verse.book}`) const entryPath = `${bookInfo.bookNumber}_${verse.chapter}/KJV_${bookInfo.bookNumber}_${verse.chapter}_${verse.verse}.mp3` const segment = { id: `${manifest.id}:${verse.id}`, resource_id: manifest.id, translation_id: 'kjv-eng-kjv2006', verse_id: verse.id, book_id: verse.book_id, book: verse.book, book_number: bookInfo.bookNumber, chapter: verse.chapter, verse: verse.verse, reference: verse.reference, granularity: 'verse', format: 'mp3', reader: 'text-to-speech', accent: 'British', audio_zip_file: zipEntry.name, audio_zip_url: zipEntry.download_url, audio_zip_bytes: zipEntry.size, audio_zip_sha: zipEntry.sha, audio_entry_path: entryPath, license: manifest.license.name, attribution: manifest.license.attribution, } segments.push(segment) const chapterKey = `${verse.book_id}.${verse.chapter}` if (!chapterPlaylists.has(chapterKey)) { chapterPlaylists.set(chapterKey, { id: `${manifest.id}:${chapterKey}`, resource_id: manifest.id, translation_id: 'kjv-eng-kjv2006', book_id: verse.book_id, book: verse.book, book_number: bookInfo.bookNumber, chapter: verse.chapter, reference: `${verse.book} ${verse.chapter}`, granularity: 'chapter', format: 'mp3-playlist', audio_zip_file: zipEntry.name, audio_zip_url: zipEntry.download_url, audio_zip_bytes: zipEntry.size, audio_zip_sha: zipEntry.sha, segment_ids: [], audio_entry_paths: [], }) } const playlist = chapterPlaylists.get(chapterKey) playlist.segment_ids.push(segment.id) playlist.audio_entry_paths.push(entryPath) if (!bookStats.has(verse.book_id)) { bookStats.set(verse.book_id, { book_id: verse.book_id, book: verse.book, book_number: bookInfo.bookNumber, chapters: new Set(), verses: 0, zipEntry, }) } const stats = bookStats.get(verse.book_id) stats.chapters.add(verse.chapter) stats.verses += 1 } const playlists = Array.from(chapterPlaylists.values()).map((playlist) => ({ ...playlist, segment_count: playlist.segment_ids.length, })) const bookPackages = Array.from(bookStats.values()) .sort((a, b) => a.book_number - b.book_number) .map((stats) => ({ id: `${manifest.id}:${String(stats.book_number).padStart(2, '0')}`, resource_id: manifest.id, translation_id: 'kjv-eng-kjv2006', book_id: stats.book_id, book: stats.book, book_number: stats.book_number, chapters: stats.chapters.size, verses: stats.verses, audio_zip_file: stats.zipEntry.name, audio_zip_url: stats.zipEntry.download_url, audio_zip_bytes: stats.zipEntry.size, audio_zip_sha: stats.zipEntry.sha, })) await mkdir(packageDir, { recursive: true }) await writeJsonl(path.join(packageDir, 'audio-segments.jsonl'), segments) await writeJsonl(path.join(packageDir, 'chapter-playlists.jsonl'), playlists) await writeJsonl(path.join(packageDir, 'book-packages.jsonl'), bookPackages) const files = { audio_segments_jsonl: await fileInfo(path.join(packageDir, 'audio-segments.jsonl'), 'audio-segments.jsonl'), chapter_playlists_jsonl: await fileInfo(path.join(packageDir, 'chapter-playlists.jsonl'), 'chapter-playlists.jsonl'), book_packages_jsonl: await fileInfo(path.join(packageDir, 'book-packages.jsonl'), 'book-packages.jsonl'), } await writeJson(path.join(packageDir, 'catalog.json'), { schema_version: 'librebible.resource-catalog.v1', project: manifest.project_name, id: manifest.id, resource_type: manifest.resource_type, title: manifest.title, short_title: manifest.short_title, abbreviation: manifest.abbreviation, alternate_ids: manifest.alternate_ids ?? [], language: manifest.language, script: manifest.script, canon: manifest.canon, contributors: manifest.contributors ?? [], features: manifest.features ?? [], attachments: manifest.attachments ?? {}, source: manifest.source, license: manifest.license, catalog_display: manifest.catalog_display, importer: manifest.importer, source_manifest_file: path.basename(manifestPath), generated_at: new Date().toISOString(), source_sha256: manifest.checks.expected_sha256, checks: manifest.checks, counts: { audio_segments: segments.length, chapter_playlists: playlists.length, book_packages: bookPackages.length, books: bookPackages.length, verses: segments.length, }, files, }) console.log(`Imported ${segments.length} KJV audio segment(s), ${playlists.length} chapter playlist(s), and ${bookPackages.length} book package(s).`) async function fetchJson(url) { const response = await fetch(url, { headers: { Accept: 'application/vnd.github+json', 'User-Agent': 'LibreBibleDataImporter', }, }) if (!response.ok) throw new Error(`GitHub API request failed ${response.status} ${response.statusText}: ${url}`) return response.json() } async function writeJsonl(filePath, rows) { await mkdir(path.dirname(filePath), { recursive: true }) await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8') } async function fileInfo(filePath, relativePath) { const stats = await stat(filePath) return { path: relativePath, bytes: stats.size, sha256: await sha256File(filePath), } }