446 lines
13 KiB
JavaScript
446 lines
13 KiB
JavaScript
import { createHash } from 'node:crypto'
|
|
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import { exists, readJson, sha256File, writeJson } from './lib.js'
|
|
|
|
const root = process.cwd()
|
|
const translationId = 'net-engnet'
|
|
const outputDir = path.join(root, 'packages', 'json', translationId)
|
|
const cacheDir = path.join(root, 'cache', translationId, 'net-notes')
|
|
const chapterCacheDir = path.join(cacheDir, 'chapters')
|
|
const noteCacheDir = path.join(cacheDir, 'notes')
|
|
const catalogPath = path.join(outputDir, 'catalog.json')
|
|
const versesPath = path.join(outputDir, 'verses.jsonl')
|
|
|
|
const verses = (await readJsonl(versesPath)).map((verse) => ({
|
|
...verse,
|
|
chapter: Number(verse.chapter),
|
|
verse: Number(verse.verse),
|
|
}))
|
|
const versesById = new Map(verses.map((verse) => [verse.id, verse]))
|
|
const chapters = chaptersFromVerses(verses)
|
|
|
|
await mkdir(chapterCacheDir, { recursive: true })
|
|
await mkdir(noteCacheDir, { recursive: true })
|
|
|
|
const markers = []
|
|
const noteFailures = []
|
|
await mapLimit(chapters, 8, async (chapter) => {
|
|
const chapterMarkers = await discoverChapterMarkers(chapter)
|
|
markers.push(...chapterMarkers)
|
|
console.log(
|
|
`${chapter.book} ${chapter.chapter}: found ${chapterMarkers.length} NET note marker(s)`,
|
|
)
|
|
})
|
|
|
|
markers.sort(compareMarker)
|
|
|
|
const notes = []
|
|
await mapLimit(markers, 14, async (marker) => {
|
|
const note = await fetchNote(marker)
|
|
if (note) notes.push(note)
|
|
})
|
|
|
|
notes.sort(compareNote)
|
|
|
|
const footnotes = []
|
|
const studyNotes = []
|
|
let footnoteId = 1
|
|
let studyNoteId = 1
|
|
const noteIndexes = new Map()
|
|
|
|
for (const note of notes) {
|
|
const isStudyNote = note.note_type === 'study_note'
|
|
const target = isStudyNote ? studyNotes : footnotes
|
|
const id = isStudyNote ? studyNoteId++ : footnoteId++
|
|
const indexKey = `${note.verse_id}:${note.note_type}`
|
|
const noteIndex = (noteIndexes.get(indexKey) ?? 0) + 1
|
|
noteIndexes.set(indexKey, noteIndex)
|
|
|
|
target.push({
|
|
id,
|
|
verse_id: note.verse_id,
|
|
translation_id: translationId,
|
|
book_id: note.book_id,
|
|
chapter: note.chapter,
|
|
verse: note.verse,
|
|
note_index: noteIndex,
|
|
note_type: note.note_type,
|
|
label: note.label,
|
|
caller: String(note.position),
|
|
reference: `${note.chapter}.${note.verse}`,
|
|
text: note.text,
|
|
source_note_position: note.position,
|
|
source_url: note.source_url,
|
|
})
|
|
}
|
|
|
|
await writeJsonl(path.join(outputDir, 'footnotes.jsonl'), footnotes)
|
|
await writeJsonl(path.join(outputDir, 'study-notes.jsonl'), studyNotes)
|
|
await writeChapterNoteFiles('footnotes', footnotes)
|
|
await writeChapterNoteFiles('study-notes', studyNotes)
|
|
|
|
const catalog = await readJson(catalogPath)
|
|
const generatedAt = new Date().toISOString()
|
|
catalog.generated_at = generatedAt
|
|
catalog.counts = {
|
|
...catalog.counts,
|
|
footnotes: footnotes.length,
|
|
study_notes: studyNotes.length,
|
|
net_note_markers: markers.length,
|
|
net_note_failures: noteFailures.length,
|
|
}
|
|
catalog.files = {
|
|
...catalog.files,
|
|
footnotes_jsonl: await packageFile(outputDir, 'footnotes.jsonl'),
|
|
study_notes_jsonl: await packageFile(outputDir, 'study-notes.jsonl'),
|
|
chapter_note_dir: {
|
|
path: 'notes-by-chapter',
|
|
layout: '<BOOK>.<CHAPTER>.footnotes.jsonl and <BOOK>.<CHAPTER>.study-notes.jsonl',
|
|
},
|
|
}
|
|
catalog.features = replaceNetNoteFeatures(catalog.features ?? [])
|
|
catalog.attachments = replaceNetNoteAttachments(catalog.attachments ?? {})
|
|
catalog.source = {
|
|
...catalog.source,
|
|
notes_url: 'https://netbible.org/resource/netNote/<reference>/<position>',
|
|
notes_api_url: 'https://labs.bible.org/api/?passage=<reference>&type=json&formatting=full',
|
|
}
|
|
catalog.catalog_display = {
|
|
...catalog.catalog_display,
|
|
primary_features: [
|
|
'Bible text',
|
|
"Strong's-linked words",
|
|
'Full NET translator notes',
|
|
'Full NET study notes',
|
|
'USFM source',
|
|
],
|
|
summary:
|
|
"Licensed/free-to-use NET Bible package using eBible USFM for text and Strong's links, plus full NET notes imported from official NET Bible note endpoints. This resource is not public domain and must follow NET Bible attribution and free-use restrictions.",
|
|
}
|
|
catalog.importer = {
|
|
...catalog.importer,
|
|
net_notes_importer: {
|
|
name: 'scripts/import-net-notes.js',
|
|
version: '0.1.0',
|
|
marker_source: 'https://labs.bible.org/api/',
|
|
note_source: 'https://netbible.org/resource/netNote/',
|
|
cache_path: 'cache/net-engnet/net-notes',
|
|
},
|
|
}
|
|
catalog.checks = {
|
|
...catalog.checks,
|
|
notes_last_checked_at: generatedAt,
|
|
notes_marker_sha256: sha256Text(markers.map((marker) => marker.cacheKey).join('\n')),
|
|
notes_failed_markers: noteFailures,
|
|
}
|
|
|
|
await writeJson(catalogPath, catalog)
|
|
|
|
console.log(`${translationId}: imported ${footnotes.length} NET footnotes`)
|
|
console.log(`${translationId}: imported ${studyNotes.length} NET study notes`)
|
|
console.log(`${translationId}: skipped ${noteFailures.length} NET note marker(s) with upstream errors`)
|
|
console.log(`${translationId}: preserved ${catalog.counts.strongs_links} Strong's links`)
|
|
|
|
async function discoverChapterMarkers(chapter) {
|
|
const cachePath = path.join(chapterCacheDir, `${chapter.bookId}-${chapter.chapter}.json`)
|
|
let rows
|
|
|
|
if (await exists(cachePath)) {
|
|
rows = await readJson(cachePath)
|
|
} else {
|
|
const url = `https://labs.bible.org/api/?passage=${encodeURIComponent(
|
|
`${chapter.book} ${chapter.chapter}`,
|
|
)}&type=json&formatting=full`
|
|
const response = await fetch(url)
|
|
if (!response.ok) {
|
|
throw new Error(`NET API marker request failed ${response.status}: ${url}`)
|
|
}
|
|
rows = await response.json()
|
|
await writeJson(cachePath, rows)
|
|
}
|
|
|
|
const chapterMarkers = []
|
|
for (const row of rows) {
|
|
const verseId = `${chapter.bookId}.${Number(row.chapter)}.${Number(row.verse)}`
|
|
if (!versesById.has(verseId)) continue
|
|
const text = String(row.text ?? '')
|
|
for (const match of text.matchAll(/<n\s+id=["']?(\d+)["']?\s*\/>/g)) {
|
|
chapterMarkers.push(markerFor(chapter, Number(row.verse), Number(match[1])))
|
|
}
|
|
}
|
|
|
|
return uniqueMarkers(chapterMarkers)
|
|
}
|
|
|
|
async function fetchNote(marker) {
|
|
const cachePath = path.join(noteCacheDir, `${marker.cacheKey}.html`)
|
|
let html
|
|
|
|
if (await exists(cachePath)) {
|
|
html = await readFile(cachePath, 'utf8')
|
|
} else {
|
|
const response = await fetch(marker.sourceUrl)
|
|
if (!response.ok) {
|
|
noteFailures.push({
|
|
verse_id: marker.verseId,
|
|
position: marker.position,
|
|
status: response.status,
|
|
source_url: marker.sourceUrl,
|
|
})
|
|
return null
|
|
}
|
|
html = await response.text()
|
|
await writeFile(cachePath, html, 'utf8')
|
|
}
|
|
|
|
const parsed = parseNoteHtml(html)
|
|
if (!parsed?.text) return null
|
|
|
|
return {
|
|
verse_id: marker.verseId,
|
|
book_id: marker.bookId,
|
|
chapter: marker.chapter,
|
|
verse: marker.verse,
|
|
position: marker.position,
|
|
note_type: parsed.noteType,
|
|
label: parsed.label,
|
|
text: parsed.text,
|
|
source_url: marker.sourceUrl,
|
|
}
|
|
}
|
|
|
|
function parseNoteHtml(html) {
|
|
if (!html || !/<div class="note">/.test(html)) return null
|
|
const typeMatch = /<span class="notetype">([^<]+)<\/span>/i.exec(html)
|
|
const sourceType = decodeHtml(typeMatch?.[1] ?? 'tn').toLowerCase()
|
|
const withoutType = html.replace(/<span class="notetype">[^<]+<\/span>/i, '')
|
|
const text = decodeHtml(
|
|
withoutType
|
|
.replace(/<\/(p|div|li|br)>/gi, ' ')
|
|
.replace(/<[^>]+>/g, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim(),
|
|
)
|
|
|
|
if (!text) return null
|
|
|
|
if (sourceType === 'sn') {
|
|
return { noteType: 'study_note', label: 'NET study note', text }
|
|
}
|
|
if (sourceType === 'tc') {
|
|
return { noteType: 'textual_critical_note', label: 'NET textual note', text }
|
|
}
|
|
return { noteType: 'translator_note', label: 'NET translator note', text }
|
|
}
|
|
|
|
function markerFor(chapter, verse, position) {
|
|
const sourceReference = `${chapter.book} ${chapter.chapter}:${verse}`
|
|
const sourceUrl = `https://netbible.org/resource/netNote/${encodeURIComponent(
|
|
sourceReference,
|
|
)}/${position}`
|
|
return {
|
|
bookId: chapter.bookId,
|
|
book: chapter.book,
|
|
chapter: chapter.chapter,
|
|
verse,
|
|
position,
|
|
verseId: `${chapter.bookId}.${chapter.chapter}.${verse}`,
|
|
cacheKey: `${chapter.bookId}-${chapter.chapter}-${verse}-${position}`,
|
|
sourceUrl,
|
|
}
|
|
}
|
|
|
|
function chaptersFromVerses(items) {
|
|
const byChapter = new Map()
|
|
for (const verse of items) {
|
|
const key = `${verse.book_id}.${verse.chapter}`
|
|
if (!byChapter.has(key)) {
|
|
byChapter.set(key, {
|
|
bookId: verse.book_id,
|
|
book: noteBookName(verse.book),
|
|
chapter: verse.chapter,
|
|
})
|
|
}
|
|
}
|
|
return [...byChapter.values()]
|
|
}
|
|
|
|
function noteBookName(book) {
|
|
if (book === 'Psalm') return 'Psalm'
|
|
return book
|
|
}
|
|
|
|
function uniqueMarkers(items) {
|
|
const seen = new Set()
|
|
return items.filter((item) => {
|
|
if (seen.has(item.cacheKey)) return false
|
|
seen.add(item.cacheKey)
|
|
return true
|
|
})
|
|
}
|
|
|
|
function compareMarker(left, right) {
|
|
return (
|
|
bookSort(left.bookId) - bookSort(right.bookId) ||
|
|
left.chapter - right.chapter ||
|
|
left.verse - right.verse ||
|
|
left.position - right.position
|
|
)
|
|
}
|
|
|
|
function compareNote(left, right) {
|
|
return (
|
|
bookSort(left.book_id) - bookSort(right.book_id) ||
|
|
left.chapter - right.chapter ||
|
|
left.verse - right.verse ||
|
|
left.position - right.position
|
|
)
|
|
}
|
|
|
|
function bookSort(bookId) {
|
|
const order = verses.findIndex((verse) => verse.book_id === bookId)
|
|
return order === -1 ? 999 : order
|
|
}
|
|
|
|
async function mapLimit(items, limit, callback) {
|
|
const queue = [...items]
|
|
const workers = Array.from({ length: Math.min(limit, queue.length) }, async () => {
|
|
while (queue.length > 0) {
|
|
const item = queue.shift()
|
|
await callback(item)
|
|
}
|
|
})
|
|
await Promise.all(workers)
|
|
}
|
|
|
|
async function readJsonl(filePath) {
|
|
const text = await readFile(filePath, 'utf8')
|
|
return text
|
|
.split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line))
|
|
}
|
|
|
|
async function writeJsonl(filePath, rows) {
|
|
await mkdir(path.dirname(filePath), { recursive: true })
|
|
await writeFile(filePath, `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, 'utf8')
|
|
}
|
|
|
|
async function writeChapterNoteFiles(kind, rows) {
|
|
const byChapter = new Map()
|
|
for (const row of rows) {
|
|
const key = `${row.book_id}.${row.chapter}`
|
|
const current = byChapter.get(key) ?? []
|
|
current.push(row)
|
|
byChapter.set(key, current)
|
|
}
|
|
|
|
const chapterDir = path.join(outputDir, 'notes-by-chapter')
|
|
if (kind === 'footnotes') {
|
|
await rm(chapterDir, { recursive: true, force: true })
|
|
}
|
|
await mkdir(chapterDir, { recursive: true })
|
|
for (const chapter of chapters) {
|
|
const key = `${chapter.bookId}.${chapter.chapter}`
|
|
await writeJsonl(path.join(chapterDir, `${key}.${kind}.jsonl`), byChapter.get(key) ?? [])
|
|
}
|
|
}
|
|
|
|
async function packageFile(outputDir, fileName) {
|
|
const filePath = path.join(outputDir, fileName)
|
|
const fileStat = await stat(filePath)
|
|
return {
|
|
path: fileName,
|
|
bytes: fileStat.size,
|
|
sha256: await sha256File(filePath),
|
|
}
|
|
}
|
|
|
|
function replaceNetNoteFeatures(features) {
|
|
const filtered = features.filter(
|
|
(feature) => !['translator-notes', 'study-notes'].includes(feature.id),
|
|
)
|
|
return [
|
|
...filtered,
|
|
{
|
|
id: 'translator-notes',
|
|
type: 'footnotes',
|
|
label: 'Full NET translator notes',
|
|
languages: ['eng'],
|
|
embedded: false,
|
|
package: 'packages/json/net-engnet/footnotes.jsonl',
|
|
source: 'Official NET Bible note endpoints.',
|
|
},
|
|
{
|
|
id: 'study-notes',
|
|
type: 'study_notes',
|
|
label: 'Full NET study notes',
|
|
languages: ['eng'],
|
|
embedded: false,
|
|
package: 'packages/json/net-engnet/study-notes.jsonl',
|
|
source: 'Official NET Bible note endpoints.',
|
|
},
|
|
]
|
|
}
|
|
|
|
function replaceNetNoteAttachments(attachments) {
|
|
const included = (attachments.included ?? []).filter(
|
|
(attachment) =>
|
|
!['net-engnet-translator-notes', 'net-engnet-study-notes'].includes(attachment.id),
|
|
)
|
|
return {
|
|
...attachments,
|
|
included: [
|
|
...included,
|
|
{
|
|
id: 'net-engnet-translator-notes',
|
|
resource_type: 'footnotes',
|
|
label: 'Full NET translator and textual notes',
|
|
relationship: 'verse-to-translator-note',
|
|
anchor_types: ['verse'],
|
|
languages: ['eng'],
|
|
source: 'Official NET Bible note endpoints.',
|
|
package: 'packages/json/net-engnet/footnotes.jsonl',
|
|
},
|
|
{
|
|
id: 'net-engnet-study-notes',
|
|
resource_type: 'study_notes',
|
|
label: 'Full NET study notes',
|
|
relationship: 'verse-to-study-note',
|
|
anchor_types: ['verse'],
|
|
languages: ['eng'],
|
|
source: 'Official NET Bible note endpoints.',
|
|
package: 'packages/json/net-engnet/study-notes.jsonl',
|
|
},
|
|
],
|
|
}
|
|
}
|
|
|
|
function decodeHtml(value) {
|
|
const named = {
|
|
amp: '&',
|
|
apos: "'",
|
|
gt: '>',
|
|
lt: '<',
|
|
nbsp: ' ',
|
|
quot: '"',
|
|
rsquo: "'",
|
|
lsquo: "'",
|
|
rdquo: '"',
|
|
ldquo: '"',
|
|
ndash: '-',
|
|
mdash: '-',
|
|
}
|
|
return value
|
|
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
|
|
.replace(/&#(\d+);/g, (_, number) => String.fromCodePoint(Number.parseInt(number, 10)))
|
|
.replace(/&([a-z]+);/gi, (match, name) => named[name.toLowerCase()] ?? match)
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
}
|
|
|
|
function sha256Text(value) {
|
|
return createHash('sha256').update(value).digest('hex')
|
|
}
|