288 lines
8.8 KiB
JavaScript
288 lines
8.8 KiB
JavaScript
import AdmZip from 'adm-zip'
|
|
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import {
|
|
cleanUsfmText,
|
|
downloadFile,
|
|
exists,
|
|
normalizeStrong,
|
|
readJson,
|
|
sha256File,
|
|
writeJson,
|
|
} from './lib.js'
|
|
|
|
const manifestPath = process.argv[2]
|
|
if (!manifestPath) {
|
|
throw new Error('Usage: node scripts/import-usfm.js sources/<manifest>.json')
|
|
}
|
|
|
|
const root = process.cwd()
|
|
const manifest = await readJson(path.resolve(root, manifestPath))
|
|
const archivePath = path.join(root, 'cache', manifest.id, path.basename(manifest.source.download_url))
|
|
const extractDir = path.join(root, 'cache', manifest.id, 'usfm')
|
|
const outputDir = path.join(root, 'packages', 'json', manifest.id)
|
|
|
|
if (!(await exists(archivePath))) {
|
|
await downloadFile(manifest.source.download_url, archivePath)
|
|
}
|
|
|
|
const archiveSha = await sha256File(archivePath)
|
|
if (archiveSha !== manifest.checks.expected_sha256.toLowerCase()) {
|
|
throw new Error(`Checksum mismatch for ${manifest.id}: ${archiveSha}`)
|
|
}
|
|
|
|
await rm(extractDir, { recursive: true, force: true })
|
|
await mkdir(extractDir, { recursive: true })
|
|
new AdmZip(archivePath).extractAllTo(extractDir, true)
|
|
|
|
const zip = new AdmZip(archivePath)
|
|
const usfmEntries = zip
|
|
.getEntries()
|
|
.filter((entry) => entry.entryName.toLowerCase().endsWith('.usfm'))
|
|
.sort((a, b) => a.entryName.localeCompare(b.entryName))
|
|
|
|
const verses = []
|
|
const strongsLinks = []
|
|
const footnotes = []
|
|
const studyNotes = []
|
|
let tokenId = 1
|
|
let footnoteId = 1
|
|
let studyNoteId = 1
|
|
|
|
for (const entry of usfmEntries) {
|
|
const filePath = path.join(extractDir, entry.entryName)
|
|
const usfm = await readFile(filePath, 'utf8')
|
|
const bookId = /\\id\s+([A-Z0-9]+)/.exec(usfm)?.[1]
|
|
const bookTitle = /\\toc2\s+(.+)/.exec(usfm)?.[1]?.trim() ?? bookId
|
|
let chapter = null
|
|
let currentVerse = null
|
|
|
|
for (const line of usfm.split(/\r?\n/)) {
|
|
const chapterMatch = /^\\c\s+(\d+)/.exec(line)
|
|
if (chapterMatch) {
|
|
flushVerse(bookId, bookTitle, currentVerse)
|
|
currentVerse = null
|
|
chapter = Number(chapterMatch[1])
|
|
continue
|
|
}
|
|
|
|
const verseMatch = /^\\v\s+(\d+)\s+([\s\S]+)/.exec(line)
|
|
if (verseMatch && chapter !== null && bookId) {
|
|
flushVerse(bookId, bookTitle, currentVerse)
|
|
currentVerse = {
|
|
chapter,
|
|
verseNumber: Number(verseMatch[1]),
|
|
raw: verseMatch[2],
|
|
}
|
|
continue
|
|
}
|
|
|
|
if (currentVerse && isVerseContinuation(line)) {
|
|
currentVerse.raw += ` ${line}`
|
|
}
|
|
}
|
|
|
|
flushVerse(bookId, bookTitle, currentVerse)
|
|
}
|
|
|
|
await mkdir(outputDir, { recursive: true })
|
|
await writeFile(
|
|
path.join(outputDir, 'verses.jsonl'),
|
|
`${verses.map((verse) => JSON.stringify(verse)).join('\n')}\n`,
|
|
'utf8',
|
|
)
|
|
await writeFile(
|
|
path.join(outputDir, 'strongs-links.jsonl'),
|
|
`${strongsLinks.map((link) => JSON.stringify(link)).join('\n')}\n`,
|
|
'utf8',
|
|
)
|
|
await writeFile(
|
|
path.join(outputDir, 'footnotes.jsonl'),
|
|
`${footnotes.map((note) => JSON.stringify(note)).join('\n')}\n`,
|
|
'utf8',
|
|
)
|
|
await writeFile(
|
|
path.join(outputDir, 'study-notes.jsonl'),
|
|
`${studyNotes.map((note) => JSON.stringify(note)).join('\n')}\n`,
|
|
'utf8',
|
|
)
|
|
|
|
const versesPackage = await packageFile(outputDir, 'verses.jsonl')
|
|
const strongsPackage = await packageFile(outputDir, 'strongs-links.jsonl')
|
|
const footnotesPackage = await packageFile(outputDir, 'footnotes.jsonl')
|
|
const studyNotesPackage = await packageFile(outputDir, 'study-notes.jsonl')
|
|
|
|
const catalog = {
|
|
schema_version: 'librebible.resource-catalog.v1',
|
|
project: manifest.project_name ?? 'LibreBible',
|
|
id: manifest.id,
|
|
resource_type: manifest.resource_type,
|
|
title: manifest.title,
|
|
short_title: manifest.short_title,
|
|
abbreviation: manifest.abbreviation,
|
|
alternate_ids: manifest.alternate_ids ?? [],
|
|
language: manifest.language,
|
|
script: manifest.script,
|
|
canon: manifest.canon,
|
|
translation: manifest.translation,
|
|
contributors: manifest.contributors ?? [],
|
|
features: manifest.features,
|
|
attachments: manifest.attachments,
|
|
source: manifest.source,
|
|
license: manifest.license,
|
|
catalog_display: manifest.catalog_display,
|
|
importer: manifest.importer,
|
|
generated_at: new Date().toISOString(),
|
|
source_sha256: archiveSha,
|
|
checks: manifest.checks,
|
|
counts: {
|
|
books: new Set(verses.map((verse) => verse.book_id)).size,
|
|
verses: verses.length,
|
|
strongs_links: strongsLinks.length,
|
|
footnotes: footnotes.length,
|
|
study_notes: studyNotes.length,
|
|
},
|
|
files: {
|
|
verses_jsonl: versesPackage,
|
|
strongs_jsonl: strongsPackage,
|
|
footnotes_jsonl: footnotesPackage,
|
|
study_notes_jsonl: studyNotesPackage,
|
|
},
|
|
}
|
|
|
|
await writeJson(path.join(outputDir, 'catalog.json'), catalog)
|
|
await writeJson(path.join(root, 'packages', 'json', 'catalog.json'), {
|
|
schema_version: 'librebible.index-catalog.v1',
|
|
project: manifest.project_name ?? 'LibreBible',
|
|
generated_at: catalog.generated_at,
|
|
resources: [
|
|
{
|
|
id: catalog.id,
|
|
resource_type: catalog.resource_type,
|
|
title: catalog.title,
|
|
short_title: catalog.short_title,
|
|
abbreviation: catalog.abbreviation,
|
|
alternate_ids: catalog.alternate_ids,
|
|
language: catalog.language,
|
|
script: catalog.script,
|
|
canon: catalog.canon,
|
|
translation: catalog.translation,
|
|
contributors: catalog.contributors,
|
|
features: catalog.features,
|
|
attachments: catalog.attachments,
|
|
source: catalog.source,
|
|
license: catalog.license,
|
|
catalog_display: catalog.catalog_display,
|
|
package_path: `packages/json/${catalog.id}/catalog.json`,
|
|
source_manifest_path: `sources/${path.basename(manifestPath)}`,
|
|
detail_doc_path: `docs/resources/${catalog.id}.md`,
|
|
counts: catalog.counts,
|
|
files: catalog.files,
|
|
checks: catalog.checks,
|
|
},
|
|
],
|
|
})
|
|
|
|
console.log(`${manifest.id}: imported ${catalog.counts.verses} verses`)
|
|
console.log(`${manifest.id}: imported ${catalog.counts.strongs_links} Strong's links`)
|
|
console.log(`${manifest.id}: imported ${catalog.counts.footnotes} footnotes`)
|
|
console.log(`${manifest.id}: imported ${catalog.counts.study_notes} study notes`)
|
|
|
|
function extractFootnotes(raw) {
|
|
return [...raw.matchAll(/\\f\s+([\s\S]*?)\\f\*/g)]
|
|
.map((match) => {
|
|
const content = match[1]
|
|
const caller = /^([+\w-]+)/.exec(content.trim())?.[1] ?? '+'
|
|
const reference = /\\fr\s+([^\\]+)/.exec(content)?.[1]?.trim() ?? ''
|
|
const label = /\\fl\s+([^\\]+)/.exec(content)?.[1]?.trim() ?? 'Footnote'
|
|
const textSource = content
|
|
.replace(/^([+\w-]+)/, '')
|
|
.replace(/\\fr\s+[^\\]+/, '')
|
|
.replace(/\\fl\s+[^\\]+/, '')
|
|
return {
|
|
caller,
|
|
reference,
|
|
label,
|
|
kind: /study note/i.test(label) ? 'study_note' : 'translator_note',
|
|
text: cleanUsfmText(textSource),
|
|
}
|
|
})
|
|
.filter((note) => note.text.length > 0 && note.text !== '[[EMPTY]]')
|
|
}
|
|
|
|
async function packageFile(outputDir, fileName) {
|
|
const filePath = path.join(outputDir, fileName)
|
|
const fileStat = await stat(filePath)
|
|
return {
|
|
path: fileName,
|
|
bytes: fileStat.size,
|
|
sha256: await sha256File(filePath),
|
|
}
|
|
}
|
|
|
|
function flushVerse(bookId, bookTitle, currentVerse) {
|
|
if (!bookId || !currentVerse) return
|
|
|
|
const { chapter, verseNumber, raw } = currentVerse
|
|
const verseId = `${bookId}.${chapter}.${verseNumber}`
|
|
const links = [...raw.matchAll(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g)]
|
|
const notes = extractFootnotes(raw)
|
|
const text = cleanUsfmText(raw.replace(/\\\+?w\s+([^|\\]+)\|strong="([^"]+)"\\\+?w\*/g, '$1'))
|
|
|
|
verses.push({
|
|
id: verseId,
|
|
translation_id: manifest.id,
|
|
book_id: bookId,
|
|
book: bookTitle,
|
|
chapter,
|
|
verse: verseNumber,
|
|
reference: `${bookTitle} ${chapter}:${verseNumber}`,
|
|
text,
|
|
})
|
|
|
|
for (const [index, link] of links.entries()) {
|
|
strongsLinks.push({
|
|
id: tokenId++,
|
|
verse_id: verseId,
|
|
translation_id: manifest.id,
|
|
book_id: bookId,
|
|
chapter,
|
|
verse: verseNumber,
|
|
position: index + 1,
|
|
surface: cleanUsfmText(link[1]),
|
|
strong_raw: link[2],
|
|
strong: normalizeStrong(link[2]),
|
|
})
|
|
}
|
|
|
|
let verseFootnoteIndex = 1
|
|
let verseStudyNoteIndex = 1
|
|
|
|
for (const note of notes) {
|
|
const target = note.kind === 'study_note' ? studyNotes : footnotes
|
|
const id = note.kind === 'study_note' ? studyNoteId++ : footnoteId++
|
|
const noteIndex =
|
|
note.kind === 'study_note' ? verseStudyNoteIndex++ : verseFootnoteIndex++
|
|
target.push({
|
|
id,
|
|
verse_id: verseId,
|
|
translation_id: manifest.id,
|
|
book_id: bookId,
|
|
chapter,
|
|
verse: verseNumber,
|
|
note_index: noteIndex,
|
|
note_type: note.kind,
|
|
label: note.label,
|
|
caller: note.caller,
|
|
reference: note.reference || `${chapter}.${verseNumber}`,
|
|
text: note.text,
|
|
})
|
|
}
|
|
}
|
|
|
|
function isVerseContinuation(line) {
|
|
if (!line.trim()) return false
|
|
if (/^\\(id|h|toc\d?|mt\d?|ms\d?|s\d?|r|rem)\b/.test(line)) return false
|
|
return !/^\\c\s+\d+/.test(line)
|
|
}
|