Add map and KJV audio resource packages
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { readJson, sha256File, writeJson } from './lib.js'
|
||||
|
||||
const manifestPath = process.argv[2]
|
||||
if (!manifestPath) {
|
||||
console.error('Usage: node scripts/import-kjv-audio-eliranwong.js sources/kjv-audio-eliranwong.json')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const root = process.cwd()
|
||||
const manifest = await readJson(path.resolve(root, manifestPath))
|
||||
const packageDir = path.join(root, 'packages', 'json', manifest.id)
|
||||
const kjvVersesPath = path.join(root, 'packages', 'json', 'kjv-eng-kjv2006', 'verses.jsonl')
|
||||
|
||||
const bookOrder = [
|
||||
['GEN', 'Genesis'],
|
||||
['EXO', 'Exodus'],
|
||||
['LEV', 'Leviticus'],
|
||||
['NUM', 'Numbers'],
|
||||
['DEU', 'Deuteronomy'],
|
||||
['JOS', 'Joshua'],
|
||||
['JDG', 'Judges'],
|
||||
['RUT', 'Ruth'],
|
||||
['1SA', '1Samuel'],
|
||||
['2SA', '2Samuel'],
|
||||
['1KI', '1Kings'],
|
||||
['2KI', '2Kings'],
|
||||
['1CH', '1Chronicles'],
|
||||
['2CH', '2Chronicles'],
|
||||
['EZR', 'Ezra'],
|
||||
['NEH', 'Nehemiah'],
|
||||
['EST', 'Esther'],
|
||||
['JOB', 'Job'],
|
||||
['PSA', 'Psalms'],
|
||||
['PRO', 'Proverbs'],
|
||||
['ECC', 'Ecclesiastes'],
|
||||
['SNG', 'SongOfSolomon'],
|
||||
['ISA', 'Isaiah'],
|
||||
['JER', 'Jeremiah'],
|
||||
['LAM', 'Lamentations'],
|
||||
['EZK', 'Ezekiel'],
|
||||
['DAN', 'Daniel'],
|
||||
['HOS', 'Hosea'],
|
||||
['JOL', 'Joel'],
|
||||
['AMO', 'Amos'],
|
||||
['OBA', 'Obadiah'],
|
||||
['JON', 'Jonah'],
|
||||
['MIC', 'Micah'],
|
||||
['NAM', 'Nahum'],
|
||||
['HAB', 'Habakkuk'],
|
||||
['ZEP', 'Zephaniah'],
|
||||
['HAG', 'Haggai'],
|
||||
['ZEC', 'Zechariah'],
|
||||
['MAL', 'Malachi'],
|
||||
['MAT', 'Matthew'],
|
||||
['MRK', 'Mark'],
|
||||
['LUK', 'Luke'],
|
||||
['JHN', 'John'],
|
||||
['ACT', 'Acts'],
|
||||
['ROM', 'Romans'],
|
||||
['1CO', '1Corinthians'],
|
||||
['2CO', '2Corinthians'],
|
||||
['GAL', 'Galatians'],
|
||||
['EPH', 'Ephesians'],
|
||||
['PHP', 'Philippians'],
|
||||
['COL', 'Colossians'],
|
||||
['1TH', '1Thessalonians'],
|
||||
['2TH', '2Thessalonians'],
|
||||
['1TI', '1Timothy'],
|
||||
['2TI', '2Timothy'],
|
||||
['TIT', 'Titus'],
|
||||
['PHM', 'Philemon'],
|
||||
['HEB', 'Hebrews'],
|
||||
['JAS', 'James'],
|
||||
['1PE', '1Peter'],
|
||||
['2PE', '2Peter'],
|
||||
['1JN', '1John'],
|
||||
['2JN', '2John'],
|
||||
['3JN', '3John'],
|
||||
['JUD', 'Jude'],
|
||||
['REV', 'Revelation'],
|
||||
]
|
||||
|
||||
const bookById = new Map(
|
||||
bookOrder.map(([bookId, audioName], index) => {
|
||||
const bookNumber = index + 1
|
||||
return [bookId, { bookId, bookNumber, audioName, zipPrefix: String(bookNumber).padStart(2, '0') }]
|
||||
}),
|
||||
)
|
||||
|
||||
const verses = (await readFile(kjvVersesPath, 'utf8'))
|
||||
.trim()
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line))
|
||||
|
||||
const audioPackageIndex = await fetchJson(manifest.source.audio_package_index_url)
|
||||
const zipEntries = audioPackageIndex
|
||||
.filter((entry) => entry.type === 'file' && entry.name.endsWith('.zip'))
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
const zipByPrefix = new Map(zipEntries.map((entry) => [entry.name.slice(0, 2), entry]))
|
||||
|
||||
const segments = []
|
||||
const chapterPlaylists = new Map()
|
||||
const bookStats = new Map()
|
||||
|
||||
for (const verse of verses) {
|
||||
const bookInfo = bookById.get(verse.book_id)
|
||||
if (!bookInfo) throw new Error(`Unsupported KJV book id: ${verse.book_id}`)
|
||||
|
||||
const zipEntry = zipByPrefix.get(bookInfo.zipPrefix)
|
||||
if (!zipEntry) throw new Error(`No audio ZIP found for ${bookInfo.zipPrefix} ${verse.book}`)
|
||||
|
||||
const entryPath = `${bookInfo.bookNumber}_${verse.chapter}/KJV_${bookInfo.bookNumber}_${verse.chapter}_${verse.verse}.mp3`
|
||||
const segment = {
|
||||
id: `${manifest.id}:${verse.id}`,
|
||||
resource_id: manifest.id,
|
||||
translation_id: 'kjv-eng-kjv2006',
|
||||
verse_id: verse.id,
|
||||
book_id: verse.book_id,
|
||||
book: verse.book,
|
||||
book_number: bookInfo.bookNumber,
|
||||
chapter: verse.chapter,
|
||||
verse: verse.verse,
|
||||
reference: verse.reference,
|
||||
granularity: 'verse',
|
||||
format: 'mp3',
|
||||
reader: 'text-to-speech',
|
||||
accent: 'British',
|
||||
audio_zip_file: zipEntry.name,
|
||||
audio_zip_url: zipEntry.download_url,
|
||||
audio_zip_bytes: zipEntry.size,
|
||||
audio_zip_sha: zipEntry.sha,
|
||||
audio_entry_path: entryPath,
|
||||
license: manifest.license.name,
|
||||
attribution: manifest.license.attribution,
|
||||
}
|
||||
segments.push(segment)
|
||||
|
||||
const chapterKey = `${verse.book_id}.${verse.chapter}`
|
||||
if (!chapterPlaylists.has(chapterKey)) {
|
||||
chapterPlaylists.set(chapterKey, {
|
||||
id: `${manifest.id}:${chapterKey}`,
|
||||
resource_id: manifest.id,
|
||||
translation_id: 'kjv-eng-kjv2006',
|
||||
book_id: verse.book_id,
|
||||
book: verse.book,
|
||||
book_number: bookInfo.bookNumber,
|
||||
chapter: verse.chapter,
|
||||
reference: `${verse.book} ${verse.chapter}`,
|
||||
granularity: 'chapter',
|
||||
format: 'mp3-playlist',
|
||||
audio_zip_file: zipEntry.name,
|
||||
audio_zip_url: zipEntry.download_url,
|
||||
audio_zip_bytes: zipEntry.size,
|
||||
audio_zip_sha: zipEntry.sha,
|
||||
segment_ids: [],
|
||||
audio_entry_paths: [],
|
||||
})
|
||||
}
|
||||
const playlist = chapterPlaylists.get(chapterKey)
|
||||
playlist.segment_ids.push(segment.id)
|
||||
playlist.audio_entry_paths.push(entryPath)
|
||||
|
||||
if (!bookStats.has(verse.book_id)) {
|
||||
bookStats.set(verse.book_id, {
|
||||
book_id: verse.book_id,
|
||||
book: verse.book,
|
||||
book_number: bookInfo.bookNumber,
|
||||
chapters: new Set(),
|
||||
verses: 0,
|
||||
zipEntry,
|
||||
})
|
||||
}
|
||||
const stats = bookStats.get(verse.book_id)
|
||||
stats.chapters.add(verse.chapter)
|
||||
stats.verses += 1
|
||||
}
|
||||
|
||||
const playlists = Array.from(chapterPlaylists.values()).map((playlist) => ({
|
||||
...playlist,
|
||||
segment_count: playlist.segment_ids.length,
|
||||
}))
|
||||
|
||||
const bookPackages = Array.from(bookStats.values())
|
||||
.sort((a, b) => a.book_number - b.book_number)
|
||||
.map((stats) => ({
|
||||
id: `${manifest.id}:${String(stats.book_number).padStart(2, '0')}`,
|
||||
resource_id: manifest.id,
|
||||
translation_id: 'kjv-eng-kjv2006',
|
||||
book_id: stats.book_id,
|
||||
book: stats.book,
|
||||
book_number: stats.book_number,
|
||||
chapters: stats.chapters.size,
|
||||
verses: stats.verses,
|
||||
audio_zip_file: stats.zipEntry.name,
|
||||
audio_zip_url: stats.zipEntry.download_url,
|
||||
audio_zip_bytes: stats.zipEntry.size,
|
||||
audio_zip_sha: stats.zipEntry.sha,
|
||||
}))
|
||||
|
||||
await mkdir(packageDir, { recursive: true })
|
||||
await writeJsonl(path.join(packageDir, 'audio-segments.jsonl'), segments)
|
||||
await writeJsonl(path.join(packageDir, 'chapter-playlists.jsonl'), playlists)
|
||||
await writeJsonl(path.join(packageDir, 'book-packages.jsonl'), bookPackages)
|
||||
|
||||
const files = {
|
||||
audio_segments_jsonl: await fileInfo(path.join(packageDir, 'audio-segments.jsonl'), 'audio-segments.jsonl'),
|
||||
chapter_playlists_jsonl: await fileInfo(path.join(packageDir, 'chapter-playlists.jsonl'), 'chapter-playlists.jsonl'),
|
||||
book_packages_jsonl: await fileInfo(path.join(packageDir, 'book-packages.jsonl'), 'book-packages.jsonl'),
|
||||
}
|
||||
|
||||
await writeJson(path.join(packageDir, 'catalog.json'), {
|
||||
schema_version: 'librebible.resource-catalog.v1',
|
||||
project: manifest.project_name,
|
||||
id: manifest.id,
|
||||
resource_type: manifest.resource_type,
|
||||
title: manifest.title,
|
||||
short_title: manifest.short_title,
|
||||
abbreviation: manifest.abbreviation,
|
||||
alternate_ids: manifest.alternate_ids ?? [],
|
||||
language: manifest.language,
|
||||
script: manifest.script,
|
||||
canon: manifest.canon,
|
||||
contributors: manifest.contributors ?? [],
|
||||
features: manifest.features ?? [],
|
||||
attachments: manifest.attachments ?? {},
|
||||
source: manifest.source,
|
||||
license: manifest.license,
|
||||
catalog_display: manifest.catalog_display,
|
||||
importer: manifest.importer,
|
||||
source_manifest_file: path.basename(manifestPath),
|
||||
generated_at: new Date().toISOString(),
|
||||
source_sha256: manifest.checks.expected_sha256,
|
||||
checks: manifest.checks,
|
||||
counts: {
|
||||
audio_segments: segments.length,
|
||||
chapter_playlists: playlists.length,
|
||||
book_packages: bookPackages.length,
|
||||
books: bookPackages.length,
|
||||
verses: segments.length,
|
||||
},
|
||||
files,
|
||||
})
|
||||
|
||||
console.log(`Imported ${segments.length} KJV audio segment(s), ${playlists.length} chapter playlist(s), and ${bookPackages.length} book package(s).`)
|
||||
|
||||
async function fetchJson(url) {
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
'User-Agent': 'LibreBibleDataImporter',
|
||||
},
|
||||
})
|
||||
if (!response.ok) throw new Error(`GitHub API request failed ${response.status} ${response.statusText}: ${url}`)
|
||||
return response.json()
|
||||
}
|
||||
|
||||
async function writeJsonl(filePath, rows) {
|
||||
await mkdir(path.dirname(filePath), { recursive: true })
|
||||
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
||||
}
|
||||
|
||||
async function fileInfo(filePath, relativePath) {
|
||||
const stats = await stat(filePath)
|
||||
return {
|
||||
path: relativePath,
|
||||
bytes: stats.size,
|
||||
sha256: await sha256File(filePath),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user