Restore source-neutral LibreBible catalog
This commit is contained in:
@@ -1,512 +0,0 @@
|
||||
import AdmZip from 'adm-zip'
|
||||
import { execFile } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import {
|
||||
classifySwordModule,
|
||||
fetchSwordConfCatalog,
|
||||
loadLicenseApproval,
|
||||
loadSwordPilotModules,
|
||||
swordPackageUrl,
|
||||
} from './sword-lib.js'
|
||||
import { cleanUsfmText, exists, sha256File, writeJson } from './lib.js'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const root = process.cwd()
|
||||
const swordCacheDir = path.join(root, 'cache', 'sword')
|
||||
const swordLibraryDir = path.join(swordCacheDir, 'library')
|
||||
const swordExportDir = path.join(swordCacheDir, 'exports')
|
||||
const swordPackageDir = path.join(swordCacheDir, 'packages')
|
||||
|
||||
const pilot = await loadSwordPilotModules()
|
||||
const licenseRegistry = await loadLicenseApproval()
|
||||
const catalog = await fetchSwordConfCatalog(pilot.repository.mods_tarball_url)
|
||||
|
||||
await mkdir(swordLibraryDir, { recursive: true })
|
||||
await mkdir(swordExportDir, { recursive: true })
|
||||
await mkdir(swordPackageDir, { recursive: true })
|
||||
|
||||
const imported = []
|
||||
const skipped = []
|
||||
|
||||
for (const item of pilot.modules) {
|
||||
try {
|
||||
if (item.import_enabled === false) {
|
||||
skipped.push(`${item.id}: import disabled - ${item.import_status ?? 'deferred'}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const catalogEntry = catalog.modules.get(item.id.toLowerCase())
|
||||
if (!catalogEntry) {
|
||||
skipped.push(`${item.id}: not found in CrossWire catalog`)
|
||||
continue
|
||||
}
|
||||
|
||||
if (item.intended_resource_type === 'translation') {
|
||||
skipped.push(`${item.id}: skipped duplicate/alternate Bible text pending edition comparison`)
|
||||
continue
|
||||
}
|
||||
|
||||
const license = classifySwordModule(catalogEntry.conf, licenseRegistry)
|
||||
if (!license.status.startsWith('approved')) {
|
||||
skipped.push(`${item.id}: license status ${license.status}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const rawzip = await downloadSwordPackage(catalogEntry.module_id)
|
||||
extractSwordPackage(rawzip.package_path)
|
||||
const exportPath = await exportSwordModule(catalogEntry.module_id)
|
||||
const records = await parseImpFile(exportPath, item.intended_resource_type, catalogEntry.conf)
|
||||
const resource = await writeSwordResource(item, catalogEntry, license, rawzip, records)
|
||||
imported.push(resource.id)
|
||||
} catch (error) {
|
||||
skipped.push(`${item.id}: import failed - ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`SWORD import complete: ${imported.length} imported, ${skipped.length} skipped.`)
|
||||
for (const id of imported) console.log(` imported: ${id}`)
|
||||
for (const reason of skipped) console.log(` skipped: ${reason}`)
|
||||
|
||||
async function downloadSwordPackage(moduleId) {
|
||||
const url = swordPackageUrl(moduleId, pilot.repository.package_base_url)
|
||||
const packagePath = path.join(swordPackageDir, `${moduleId}.zip`)
|
||||
|
||||
if (!(await exists(packagePath))) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Unable to download SWORD package ${url}: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
await writeFile(packagePath, Buffer.from(await response.arrayBuffer()))
|
||||
}
|
||||
|
||||
const checksum = await sha256File(packagePath)
|
||||
const fileStat = await stat(packagePath)
|
||||
return {
|
||||
package_url: url,
|
||||
package_path: packagePath,
|
||||
bytes: fileStat.size,
|
||||
sha256: checksum,
|
||||
}
|
||||
}
|
||||
|
||||
function extractSwordPackage(packagePath) {
|
||||
const archive = new AdmZip(packagePath)
|
||||
archive.extractAllTo(swordLibraryDir, true)
|
||||
}
|
||||
|
||||
async function exportSwordModule(moduleId) {
|
||||
const outputPath = path.join(swordExportDir, `${moduleId}.imp`)
|
||||
if (await exists(outputPath)) {
|
||||
const outputStat = await stat(outputPath)
|
||||
if (outputStat.size > 0) {
|
||||
return outputPath
|
||||
}
|
||||
}
|
||||
|
||||
const shell = [
|
||||
'set -e',
|
||||
'export DEBIAN_FRONTEND=noninteractive',
|
||||
'apt-get update >/dev/null',
|
||||
'apt-get install -y libsword-utils >/dev/null',
|
||||
`SWORD_PATH=/workspace/cache/sword/library mod2imp ${shellQuote(moduleId)} > /workspace/cache/sword/exports/${shellQuote(moduleId)}.imp`,
|
||||
].join('\n')
|
||||
|
||||
await execFileAsync('docker', [
|
||||
'run',
|
||||
'--rm',
|
||||
'-v',
|
||||
`${root}:/workspace`,
|
||||
'-w',
|
||||
'/workspace',
|
||||
'ubuntu:26.04',
|
||||
'bash',
|
||||
'-lc',
|
||||
shell,
|
||||
], {
|
||||
maxBuffer: 20 * 1024 * 1024,
|
||||
timeout: 120_000,
|
||||
})
|
||||
|
||||
if (!(await exists(outputPath))) {
|
||||
throw new Error(`SWORD export did not create ${outputPath}`)
|
||||
}
|
||||
|
||||
return outputPath
|
||||
}
|
||||
|
||||
async function parseImpFile(filePath, resourceType, conf) {
|
||||
const text = await readFile(filePath, 'utf8')
|
||||
const records = []
|
||||
let currentKey = null
|
||||
let body = []
|
||||
|
||||
function flush() {
|
||||
if (!currentKey) return
|
||||
const raw = body.join('\n').trim()
|
||||
if (!raw) {
|
||||
currentKey = null
|
||||
body = []
|
||||
return
|
||||
}
|
||||
|
||||
if (resourceType === 'commentary') {
|
||||
const record = commentaryRecord(currentKey, raw, conf)
|
||||
if (record) records.push(record)
|
||||
} else {
|
||||
const record = dictionaryRecord(currentKey, raw, conf)
|
||||
if (record) records.push(record)
|
||||
}
|
||||
|
||||
currentKey = null
|
||||
body = []
|
||||
}
|
||||
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (line.startsWith('$$$')) {
|
||||
flush()
|
||||
currentKey = line.slice(3).trim()
|
||||
} else {
|
||||
body.push(line)
|
||||
}
|
||||
}
|
||||
|
||||
flush()
|
||||
return records
|
||||
}
|
||||
|
||||
async function writeSwordResource(item, catalogEntry, license, rawzip, records) {
|
||||
const conf = catalogEntry.conf
|
||||
const id = resourceIdFor(catalogEntry.module_id)
|
||||
const outputDir = path.join(root, 'packages', 'json', id)
|
||||
const fileName = item.intended_resource_type === 'commentary' ? 'commentary.jsonl' : 'entries.jsonl'
|
||||
const filePath = path.join(outputDir, fileName)
|
||||
|
||||
await rm(outputDir, { recursive: true, force: true })
|
||||
await mkdir(outputDir, { recursive: true })
|
||||
await writeFile(filePath, `${records.map((record) => JSON.stringify(record)).join('\n')}\n`, 'utf8')
|
||||
|
||||
const packageFile = await describePackageFile(outputDir, fileName)
|
||||
const manifest = sourceManifestFor(item, catalogEntry, license, rawzip, id, fileName, records)
|
||||
const catalog = packageCatalogFor(manifest, catalogEntry, rawzip, records, packageFile)
|
||||
|
||||
await writeJson(path.join(root, 'sources', `${id}.json`), manifest)
|
||||
await writeJson(path.join(outputDir, 'catalog.json'), catalog)
|
||||
|
||||
return { id, records: records.length }
|
||||
}
|
||||
|
||||
function sourceManifestFor(item, catalogEntry, license, rawzip, id, fileName, records) {
|
||||
const conf = catalogEntry.conf
|
||||
const resourceType = item.intended_resource_type
|
||||
const title = conf.Description ?? catalogEntry.module_id
|
||||
const licenseName = conf.DistributionLicense ?? license.rule ?? 'Unknown'
|
||||
|
||||
return {
|
||||
id,
|
||||
resource_type: resourceType,
|
||||
project_name: 'LibreBible',
|
||||
title,
|
||||
short_title: title,
|
||||
abbreviation: conf.Abbreviation ?? catalogEntry.module_id,
|
||||
alternate_ids: [catalogEntry.module_id],
|
||||
language: {
|
||||
code: conf.Lang ?? 'en',
|
||||
name: conf.Lang === 'en' || !conf.Lang ? 'English' : conf.Lang,
|
||||
},
|
||||
script: 'Latn',
|
||||
canon: {
|
||||
scope: conf.Versification ?? 'n/a',
|
||||
notes: resourceType === 'commentary' ? 'Commentary entries are keyed to SWORD Bible references.' : 'Dictionary entries are keyed by SWORD lexical keys.',
|
||||
},
|
||||
contributors: contributorsFor(conf, resourceType),
|
||||
features: featuresFor(resourceType, id, fileName, catalogEntry),
|
||||
attachments: attachmentsFor(resourceType, id, fileName),
|
||||
source: {
|
||||
provider: 'CrossWire Bible Society',
|
||||
url: `https://crosswire.org/sword/modules/ModInfo.jsp?modName=${encodeURIComponent(catalogEntry.module_id)}`,
|
||||
download_url: rawzip.package_url,
|
||||
format: 'sword-rawzip',
|
||||
upstream_id: catalogEntry.module_id,
|
||||
upstream_last_updated: conf.SwordVersionDate,
|
||||
sword: {
|
||||
module_id: catalogEntry.module_id,
|
||||
conf_file: catalogEntry.conf_file,
|
||||
mod_drv: conf.ModDrv,
|
||||
source_type: conf.SourceType,
|
||||
versification: conf.Versification,
|
||||
version: conf.Version,
|
||||
data_path: conf.DataPath,
|
||||
text_source: conf.TextSource,
|
||||
},
|
||||
},
|
||||
license: {
|
||||
name: licenseName,
|
||||
redistribution: license.redistribution === true,
|
||||
commercial_use: license.commercial_use,
|
||||
share_alike: license.share_alike === true,
|
||||
source_license_url: `https://crosswire.org/ftpmirror/pub/sword/raw/mods.d/${catalogEntry.module_id.toLowerCase()}.conf`,
|
||||
attribution: attributionFor(conf, catalogEntry.module_id),
|
||||
jurisdiction_notes: conf.Copyright ?? 'No jurisdiction-specific restriction was found in the SWORD module metadata.',
|
||||
restricted_notes: license.notes,
|
||||
},
|
||||
checks: {
|
||||
expected_sha256: rawzip.sha256,
|
||||
last_checked_at: new Date().toISOString(),
|
||||
},
|
||||
importer: {
|
||||
name: 'scripts/import-sword.js',
|
||||
version: '0.1.0',
|
||||
engine: 'CrossWire SWORD mod2imp via Docker/Ubuntu libsword-utils',
|
||||
},
|
||||
packages: {
|
||||
json_catalog: `packages/json/${id}/catalog.json`,
|
||||
entries_jsonl: `packages/json/${id}/${fileName}`,
|
||||
},
|
||||
catalog_display: {
|
||||
primary_features: resourceType === 'commentary' ? ['Commentary entries', 'Verse/range anchors', 'SWORD source'] : ['Dictionary entries', 'Lexical anchors', 'SWORD source'],
|
||||
summary: summaryFor(resourceType, title, records.length, catalogEntry.module_id),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function packageCatalogFor(manifest, catalogEntry, rawzip, records, packageFile) {
|
||||
const fileKey = manifest.resource_type === 'commentary' ? 'commentary_jsonl' : 'entries_jsonl'
|
||||
return {
|
||||
schema_version: 'librebible.resource-catalog.v1',
|
||||
project: 'LibreBible',
|
||||
id: manifest.id,
|
||||
source_manifest_file: `${manifest.id}.json`,
|
||||
resource_type: manifest.resource_type,
|
||||
title: manifest.title,
|
||||
short_title: manifest.short_title,
|
||||
abbreviation: manifest.abbreviation,
|
||||
alternate_ids: manifest.alternate_ids,
|
||||
language: manifest.language,
|
||||
script: manifest.script,
|
||||
canon: manifest.canon,
|
||||
contributors: manifest.contributors,
|
||||
features: manifest.features,
|
||||
attachments: manifest.attachments,
|
||||
source: manifest.source,
|
||||
license: manifest.license,
|
||||
catalog_display: manifest.catalog_display,
|
||||
importer: manifest.importer,
|
||||
generated_at: new Date().toISOString(),
|
||||
source_sha256: rawzip.sha256,
|
||||
checks: manifest.checks,
|
||||
counts: {
|
||||
entries: records.length,
|
||||
commentary_entries: manifest.resource_type === 'commentary' ? records.length : undefined,
|
||||
dictionary_entries: manifest.resource_type === 'dictionary' ? records.length : undefined,
|
||||
},
|
||||
files: {
|
||||
[fileKey]: packageFile,
|
||||
source_rawzip: {
|
||||
path: path.basename(rawzip.package_path),
|
||||
bytes: rawzip.bytes,
|
||||
sha256: rawzip.sha256,
|
||||
},
|
||||
},
|
||||
sword: manifest.source.sword,
|
||||
raw_conf_sha256: createHash('sha256').update(catalogEntry.conf_text).digest('hex'),
|
||||
}
|
||||
}
|
||||
|
||||
async function describePackageFile(outputDir, fileName) {
|
||||
const filePath = path.join(outputDir, fileName)
|
||||
const fileStat = await stat(filePath)
|
||||
return {
|
||||
path: fileName,
|
||||
bytes: fileStat.size,
|
||||
sha256: await sha256File(filePath),
|
||||
}
|
||||
}
|
||||
|
||||
function commentaryRecord(key, raw, conf) {
|
||||
const parsed = parseBibleKey(key)
|
||||
const text = stripMarkup(raw)
|
||||
if (!text) return null
|
||||
return {
|
||||
id: stableId(`${conf.module_id}:${key}`),
|
||||
resource_id: resourceIdFor(conf.module_id),
|
||||
module_id: conf.module_id,
|
||||
key,
|
||||
anchor_type: parsed ? 'verse' : 'module_key',
|
||||
reference: parsed?.reference ?? key,
|
||||
book: parsed?.book,
|
||||
chapter: parsed?.chapter,
|
||||
verse: parsed?.verse,
|
||||
raw_markup: raw,
|
||||
text,
|
||||
}
|
||||
}
|
||||
|
||||
function dictionaryRecord(key, raw, conf) {
|
||||
const text = stripMarkup(raw)
|
||||
if (!text) return null
|
||||
const prefix = conf.module_id === 'StrongsGreek' ? 'G' : conf.module_id === 'StrongsHebrew' ? 'H' : ''
|
||||
const number = /^\d+$/.test(key) ? Number(key) : null
|
||||
const lexicalKey = prefix && number !== null ? `${prefix}${number}` : key
|
||||
return {
|
||||
id: stableId(`${conf.module_id}:${key}`),
|
||||
resource_id: resourceIdFor(conf.module_id),
|
||||
module_id: conf.module_id,
|
||||
key,
|
||||
lexical_key: lexicalKey,
|
||||
anchor_type: prefix ? 'strongs_number' : 'dictionary_key',
|
||||
raw_markup: raw,
|
||||
text,
|
||||
}
|
||||
}
|
||||
|
||||
function parseBibleKey(key) {
|
||||
const match = /^(.+?)\s+(\d+):(\d+)$/.exec(key)
|
||||
if (!match) return null
|
||||
return {
|
||||
book: match[1],
|
||||
chapter: Number(match[2]),
|
||||
verse: Number(match[3]),
|
||||
reference: key,
|
||||
}
|
||||
}
|
||||
|
||||
function stripMarkup(raw) {
|
||||
return cleanUsfmText(raw
|
||||
.replace(/<reference[^>]*>(.*?)<\/reference>/gis, '$1')
|
||||
.replace(/<ref[^>]*>(.*?)<\/ref>/gis, '$1')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>'))
|
||||
}
|
||||
|
||||
function resourceIdFor(moduleId) {
|
||||
return `sword-${moduleId.replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase()}`
|
||||
}
|
||||
|
||||
function stableId(value) {
|
||||
return createHash('sha1').update(value).digest('hex')
|
||||
}
|
||||
|
||||
function featuresFor(resourceType, id, fileName, catalogEntry) {
|
||||
if (resourceType === 'commentary') {
|
||||
return [
|
||||
{
|
||||
id: 'commentary-entries',
|
||||
type: 'commentary',
|
||||
label: 'Commentary entries',
|
||||
languages: ['eng'],
|
||||
embedded: true,
|
||||
package: `packages/json/${id}/${fileName}`,
|
||||
},
|
||||
{
|
||||
id: 'sword-source',
|
||||
type: 'source_format',
|
||||
label: 'SWORD source',
|
||||
format: catalogEntry.conf.SourceType ?? catalogEntry.conf.ModDrv,
|
||||
embedded: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
id: 'dictionary-entries',
|
||||
type: 'dictionary',
|
||||
label: 'Dictionary entries',
|
||||
languages: ['eng'],
|
||||
systems: catalogEntry.module_id.startsWith('Strongs') ? ['strongs'] : undefined,
|
||||
embedded: true,
|
||||
package: `packages/json/${id}/${fileName}`,
|
||||
},
|
||||
{
|
||||
id: 'sword-source',
|
||||
type: 'source_format',
|
||||
label: 'SWORD source',
|
||||
format: catalogEntry.conf.SourceType ?? catalogEntry.conf.ModDrv,
|
||||
embedded: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
function attachmentsFor(resourceType, id, fileName) {
|
||||
if (resourceType === 'commentary') {
|
||||
return {
|
||||
included: [
|
||||
{
|
||||
id: `${id}-commentary`,
|
||||
resource_type: 'commentary',
|
||||
label: 'Commentary entries',
|
||||
relationship: 'range-to-commentary',
|
||||
anchor_types: ['book', 'chapter', 'verse', 'verse_range'],
|
||||
languages: ['eng'],
|
||||
source: 'CrossWire SWORD module export.',
|
||||
package: `packages/json/${id}/${fileName}`,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
included: [
|
||||
{
|
||||
id: `${id}-dictionary`,
|
||||
resource_type: 'dictionary',
|
||||
label: 'Dictionary entries',
|
||||
relationship: 'key-to-definition',
|
||||
anchor_types: ['strongs_number', 'topic', 'dictionary_key'],
|
||||
languages: ['eng'],
|
||||
systems: id.includes('strongs') ? ['strongs'] : undefined,
|
||||
source: 'CrossWire SWORD module export.',
|
||||
package: `packages/json/${id}/${fileName}`,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
function contributorsFor(conf, resourceType) {
|
||||
const contributors = [
|
||||
{
|
||||
name: 'CrossWire Bible Society',
|
||||
role: 'module provider',
|
||||
},
|
||||
]
|
||||
|
||||
if (resourceType === 'commentary' && /Matthew Henry/i.test(conf.Description ?? '')) {
|
||||
contributors.unshift({ name: 'Matthew Henry', role: 'commentary author' })
|
||||
} else if (resourceType === 'commentary' && /Jamieson/i.test(conf.Description ?? '')) {
|
||||
contributors.unshift({ name: 'Robert Jamieson, A. R. Fausset, and David Brown', role: 'commentary authors' })
|
||||
} else if (/Strong/i.test(conf.Description ?? '')) {
|
||||
contributors.unshift({ name: 'James Strong', role: 'dictionary author' })
|
||||
} else if (/Nave/i.test(conf.Description ?? '')) {
|
||||
contributors.unshift({ name: 'Orville J. Nave', role: 'topical dictionary author' })
|
||||
}
|
||||
|
||||
return contributors
|
||||
}
|
||||
|
||||
function attributionFor(conf, moduleId) {
|
||||
const parts = [
|
||||
conf.Description,
|
||||
conf.TextSource ? `Text source: ${conf.TextSource}` : null,
|
||||
`SWORD module: ${moduleId}`,
|
||||
'Provided by CrossWire Bible Society.',
|
||||
]
|
||||
return parts.filter(Boolean).join(' ')
|
||||
}
|
||||
|
||||
function summaryFor(resourceType, title, count, moduleId) {
|
||||
return resourceType === 'commentary'
|
||||
? `${title}, imported from the CrossWire ${moduleId} SWORD module with ${count} commentary entries.`
|
||||
: `${title}, imported from the CrossWire ${moduleId} SWORD module with ${count} dictionary entries.`
|
||||
}
|
||||
|
||||
function shellQuote(value) {
|
||||
return `'${String(value).replace(/'/g, `'\\''`)}'`
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
import {
|
||||
classifySwordModule,
|
||||
fetchSwordConfCatalog,
|
||||
loadLicenseApproval,
|
||||
loadSwordPilotModules,
|
||||
swordPackageUrl,
|
||||
writeSwordAudit,
|
||||
} from './sword-lib.js'
|
||||
|
||||
const pilot = await loadSwordPilotModules()
|
||||
const licenseRegistry = await loadLicenseApproval()
|
||||
const catalog = await fetchSwordConfCatalog(pilot.repository.mods_tarball_url)
|
||||
|
||||
const modules = []
|
||||
|
||||
for (const item of pilot.modules) {
|
||||
const catalogEntry = catalog.modules.get(item.id.toLowerCase())
|
||||
|
||||
if (!catalogEntry) {
|
||||
modules.push({
|
||||
id: item.id,
|
||||
intended_resource_type: item.intended_resource_type,
|
||||
found: false,
|
||||
license_status: 'missing',
|
||||
review_notes: 'Module was not found in the CrossWire catalog.',
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
const conf = catalogEntry.conf
|
||||
const license = classifySwordModule(conf, licenseRegistry)
|
||||
modules.push({
|
||||
id: catalogEntry.module_id,
|
||||
intended_resource_type: item.intended_resource_type,
|
||||
found: true,
|
||||
title: conf.Description ?? catalogEntry.module_id,
|
||||
abbreviation: conf.Abbreviation ?? catalogEntry.module_id,
|
||||
lang: conf.Lang ?? 'n/a',
|
||||
mod_drv: conf.ModDrv,
|
||||
source_type: conf.SourceType,
|
||||
version: conf.Version,
|
||||
sword_version_date: conf.SwordVersionDate,
|
||||
distribution_license: conf.DistributionLicense ?? 'n/a',
|
||||
text_source: conf.TextSource ?? 'n/a',
|
||||
package_url: swordPackageUrl(catalogEntry.module_id, pilot.repository.package_base_url),
|
||||
license_status: license.status,
|
||||
license_rule: license.rule,
|
||||
redistribution: license.redistribution ?? false,
|
||||
commercial_use: license.commercial_use ?? null,
|
||||
share_alike: license.share_alike ?? false,
|
||||
import_enabled: item.import_enabled !== false,
|
||||
import_status: item.import_status ?? (item.import_enabled === false ? 'deferred' : 'candidate'),
|
||||
review_notes: license.notes,
|
||||
reason: item.reason,
|
||||
})
|
||||
}
|
||||
|
||||
const audit = {
|
||||
schema_version: 'librebible.sword-module-audit.v1',
|
||||
generated_at: new Date().toISOString(),
|
||||
repository: pilot.repository,
|
||||
catalog_sha256: catalog.sha256,
|
||||
modules,
|
||||
}
|
||||
|
||||
await writeSwordAudit(audit)
|
||||
|
||||
const approved = modules.filter((module) => module.license_status.startsWith('approved')).length
|
||||
const review = modules.filter((module) => module.license_status === 'review').length
|
||||
const rejected = modules.filter((module) => module.license_status === 'rejected').length
|
||||
|
||||
console.log(`SWORD audit complete: ${modules.length} module(s), ${approved} approved, ${review} review, ${rejected} rejected.`)
|
||||
@@ -1,217 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { exists, readJson, writeJson } from './lib.js'
|
||||
|
||||
export const swordCacheDir = path.join(process.cwd(), 'cache', 'sword')
|
||||
export const defaultModsTarballUrl = 'https://crosswire.org/ftpmirror/pub/sword/raw/mods.d.tar.gz'
|
||||
export const defaultPackageBaseUrl = 'https://crosswire.org/ftpmirror/pub/sword/packages/rawzip'
|
||||
|
||||
export async function fetchSwordConfCatalog(url = defaultModsTarballUrl) {
|
||||
await mkdir(swordCacheDir, { recursive: true })
|
||||
const tarballPath = path.join(swordCacheDir, 'mods.d.tar.gz')
|
||||
|
||||
if (!(await exists(tarballPath))) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Unable to fetch SWORD module catalog ${url}: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
await writeFile(tarballPath, Buffer.from(await response.arrayBuffer()))
|
||||
}
|
||||
|
||||
const tarball = await readFile(tarballPath)
|
||||
const sha256 = createHash('sha256').update(tarball).digest('hex')
|
||||
const tar = gunzipSync(tarball)
|
||||
const modules = new Map()
|
||||
|
||||
for (const entry of readTarEntries(tar)) {
|
||||
if (!entry.name.endsWith('.conf')) continue
|
||||
const text = entry.content.toString('utf8')
|
||||
const conf = parseSwordConf(text)
|
||||
if (conf.module_id) {
|
||||
modules.set(conf.module_id.toLowerCase(), {
|
||||
module_id: conf.module_id,
|
||||
conf_file: entry.name,
|
||||
conf_text: text,
|
||||
conf,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source_url: url,
|
||||
tarball_path: tarballPath,
|
||||
sha256,
|
||||
modules,
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadSwordPilotModules(filePath = path.join(process.cwd(), 'registries', 'sword-pilot-modules.json')) {
|
||||
return readJson(filePath)
|
||||
}
|
||||
|
||||
export async function loadLicenseApproval(filePath = path.join(process.cwd(), 'registries', 'license-approval.json')) {
|
||||
return readJson(filePath)
|
||||
}
|
||||
|
||||
export function classifySwordModule(conf, licenseRegistry) {
|
||||
const licenseText = [
|
||||
conf.DistributionLicense,
|
||||
conf.Copyright,
|
||||
conf.About,
|
||||
conf.ShortPromo,
|
||||
].filter(Boolean).join('\n').toLowerCase()
|
||||
|
||||
for (const rule of licenseRegistry.rejected ?? []) {
|
||||
if (licenseText.includes(rule.match.toLowerCase())) {
|
||||
return { status: rule.status, rule: rule.match, notes: rule.notes }
|
||||
}
|
||||
}
|
||||
|
||||
for (const rule of licenseRegistry.approved ?? []) {
|
||||
if (licenseText.includes(rule.match.toLowerCase())) {
|
||||
return {
|
||||
status: rule.status,
|
||||
rule: rule.match,
|
||||
redistribution: rule.redistribution,
|
||||
commercial_use: rule.commercial_use,
|
||||
share_alike: rule.share_alike,
|
||||
notes: rule.notes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const rule of licenseRegistry.review ?? []) {
|
||||
if (licenseText.includes(rule.match.toLowerCase())) {
|
||||
return { status: rule.status, rule: rule.match, notes: rule.notes }
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'review',
|
||||
rule: null,
|
||||
notes: 'No approval rule matched the module license metadata.',
|
||||
}
|
||||
}
|
||||
|
||||
export function swordPackageUrl(moduleId, packageBaseUrl = defaultPackageBaseUrl) {
|
||||
return `${packageBaseUrl.replace(/\/$/, '')}/${moduleId}.zip`
|
||||
}
|
||||
|
||||
export async function writeSwordAudit(outputs) {
|
||||
await writeJson(path.join(process.cwd(), 'audits', 'sword', 'module-audit.json'), outputs)
|
||||
await mkdir(path.join(process.cwd(), 'docs'), { recursive: true })
|
||||
await writeFile(path.join(process.cwd(), 'docs', 'sword-module-audit.md'), swordAuditMarkdown(outputs), 'utf8')
|
||||
}
|
||||
|
||||
export function parseSwordConf(input) {
|
||||
const conf = {}
|
||||
let moduleId = null
|
||||
let currentKey = null
|
||||
|
||||
for (const rawLine of input.split(/\r?\n/)) {
|
||||
const line = rawLine.trim()
|
||||
if (!line || line.startsWith('#')) continue
|
||||
|
||||
const section = /^\[([^\]]+)\]$/.exec(line)
|
||||
if (section) {
|
||||
moduleId = section[1]
|
||||
conf.module_id = moduleId
|
||||
currentKey = null
|
||||
continue
|
||||
}
|
||||
|
||||
const kv = /^([^=]+)=(.*)$/.exec(line)
|
||||
if (kv) {
|
||||
const key = kv[1].trim()
|
||||
const value = cleanSwordValue(kv[2].trim())
|
||||
if (conf[key] === undefined) {
|
||||
conf[key] = value
|
||||
} else if (Array.isArray(conf[key])) {
|
||||
conf[key].push(value)
|
||||
} else {
|
||||
conf[key] = [conf[key], value]
|
||||
}
|
||||
currentKey = key
|
||||
continue
|
||||
}
|
||||
|
||||
if (currentKey && typeof conf[currentKey] === 'string') {
|
||||
conf[currentKey] = `${conf[currentKey]}\n${cleanSwordValue(line)}`
|
||||
}
|
||||
}
|
||||
|
||||
return conf
|
||||
}
|
||||
|
||||
function cleanSwordValue(value) {
|
||||
return value.replace(/\\par\s*/g, '\n').replace(/\s+\n/g, '\n').trim()
|
||||
}
|
||||
|
||||
function readTarEntries(buffer) {
|
||||
const entries = []
|
||||
let offset = 0
|
||||
|
||||
while (offset + 512 <= buffer.length) {
|
||||
const header = buffer.subarray(offset, offset + 512)
|
||||
if (header.every((byte) => byte === 0)) break
|
||||
|
||||
const name = readNullTerminated(header.subarray(0, 100))
|
||||
const sizeText = readNullTerminated(header.subarray(124, 136)).trim()
|
||||
const size = Number.parseInt(sizeText || '0', 8)
|
||||
const contentStart = offset + 512
|
||||
const contentEnd = contentStart + size
|
||||
entries.push({
|
||||
name,
|
||||
size,
|
||||
content: buffer.subarray(contentStart, contentEnd),
|
||||
})
|
||||
|
||||
offset = contentStart + Math.ceil(size / 512) * 512
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
function readNullTerminated(buffer) {
|
||||
const zero = buffer.indexOf(0)
|
||||
return buffer.subarray(0, zero === -1 ? buffer.length : zero).toString('utf8')
|
||||
}
|
||||
|
||||
function swordAuditMarkdown(audit) {
|
||||
const lines = [
|
||||
'# CrossWire SWORD Module Audit',
|
||||
'',
|
||||
'This file is generated from the CrossWire SWORD module catalog and the local LibreBible license approval registry.',
|
||||
'',
|
||||
`- Repository: ${audit.repository.name}`,
|
||||
`- Catalog URL: ${audit.repository.mods_tarball_url}`,
|
||||
`- Catalog SHA-256: \`${audit.catalog_sha256}\``,
|
||||
`- Generated: ${audit.generated_at}`,
|
||||
'',
|
||||
'| Module | Intended Type | SWORD Type | Source Type | License | License Status | Import Status | Package | Notes |',
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- | --- |',
|
||||
]
|
||||
|
||||
for (const module of audit.modules) {
|
||||
lines.push([
|
||||
module.id,
|
||||
module.intended_resource_type,
|
||||
module.mod_drv ?? 'n/a',
|
||||
module.source_type ?? 'n/a',
|
||||
module.distribution_license ?? 'n/a',
|
||||
module.license_status,
|
||||
module.import_status ?? (module.import_enabled === false ? 'deferred' : 'candidate'),
|
||||
module.package_url ? `[zip](${module.package_url})` : 'n/a',
|
||||
module.review_notes.replace(/\|/g, '/'),
|
||||
].join(' | ').replace(/^/, '| ').replace(/$/, ' |'))
|
||||
}
|
||||
|
||||
lines.push('')
|
||||
lines.push('## Next Import Gate')
|
||||
lines.push('')
|
||||
lines.push('A SWORD module should not move from this audit into `packages/json/` until its license metadata is approved and the importer can convert the module content into LibreBible package formats with checksums.')
|
||||
|
||||
return `${lines.join('\n')}\n`
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
import { fetchSwordConfCatalog } from './sword-lib.js'
|
||||
|
||||
const catalog = await fetchSwordConfCatalog()
|
||||
const modules = [...catalog.modules.values()]
|
||||
.map((entry) => ({
|
||||
id: entry.module_id,
|
||||
description: scalar(entry.conf.Description),
|
||||
mod_drv: scalar(entry.conf.ModDrv),
|
||||
source_type: scalar(entry.conf.SourceType),
|
||||
license: scalar(entry.conf.DistributionLicense),
|
||||
lang: scalar(entry.conf.Lang),
|
||||
}))
|
||||
.sort((a, b) => a.id.localeCompare(b.id))
|
||||
|
||||
for (const module of modules) {
|
||||
console.log([
|
||||
module.id.padEnd(24),
|
||||
module.mod_drv.padEnd(8),
|
||||
module.source_type.padEnd(8),
|
||||
module.lang.padEnd(6),
|
||||
module.license.padEnd(24),
|
||||
module.description,
|
||||
].join(' ').trimEnd())
|
||||
}
|
||||
|
||||
console.log(`\n${modules.length} SWORD module(s) listed from ${catalog.source_url}`)
|
||||
|
||||
function scalar(value) {
|
||||
if (Array.isArray(value)) return value.join(', ')
|
||||
return value ?? ''
|
||||
}
|
||||
Reference in New Issue
Block a user