Phase 6 add CrossWire SWORD imports
This commit is contained in:
@@ -0,0 +1,217 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { exists, readJson, writeJson } from './lib.js'
|
||||
|
||||
export const swordCacheDir = path.join(process.cwd(), 'cache', 'sword')
|
||||
export const defaultModsTarballUrl = 'https://crosswire.org/ftpmirror/pub/sword/raw/mods.d.tar.gz'
|
||||
export const defaultPackageBaseUrl = 'https://crosswire.org/ftpmirror/pub/sword/packages/rawzip'
|
||||
|
||||
export async function fetchSwordConfCatalog(url = defaultModsTarballUrl) {
|
||||
await mkdir(swordCacheDir, { recursive: true })
|
||||
const tarballPath = path.join(swordCacheDir, 'mods.d.tar.gz')
|
||||
|
||||
if (!(await exists(tarballPath))) {
|
||||
const response = await fetch(url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Unable to fetch SWORD module catalog ${url}: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
await writeFile(tarballPath, Buffer.from(await response.arrayBuffer()))
|
||||
}
|
||||
|
||||
const tarball = await readFile(tarballPath)
|
||||
const sha256 = createHash('sha256').update(tarball).digest('hex')
|
||||
const tar = gunzipSync(tarball)
|
||||
const modules = new Map()
|
||||
|
||||
for (const entry of readTarEntries(tar)) {
|
||||
if (!entry.name.endsWith('.conf')) continue
|
||||
const text = entry.content.toString('utf8')
|
||||
const conf = parseSwordConf(text)
|
||||
if (conf.module_id) {
|
||||
modules.set(conf.module_id.toLowerCase(), {
|
||||
module_id: conf.module_id,
|
||||
conf_file: entry.name,
|
||||
conf_text: text,
|
||||
conf,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source_url: url,
|
||||
tarball_path: tarballPath,
|
||||
sha256,
|
||||
modules,
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadSwordPilotModules(filePath = path.join(process.cwd(), 'registries', 'sword-pilot-modules.json')) {
|
||||
return readJson(filePath)
|
||||
}
|
||||
|
||||
export async function loadLicenseApproval(filePath = path.join(process.cwd(), 'registries', 'license-approval.json')) {
|
||||
return readJson(filePath)
|
||||
}
|
||||
|
||||
export function classifySwordModule(conf, licenseRegistry) {
|
||||
const licenseText = [
|
||||
conf.DistributionLicense,
|
||||
conf.Copyright,
|
||||
conf.About,
|
||||
conf.ShortPromo,
|
||||
].filter(Boolean).join('\n').toLowerCase()
|
||||
|
||||
for (const rule of licenseRegistry.rejected ?? []) {
|
||||
if (licenseText.includes(rule.match.toLowerCase())) {
|
||||
return { status: rule.status, rule: rule.match, notes: rule.notes }
|
||||
}
|
||||
}
|
||||
|
||||
for (const rule of licenseRegistry.approved ?? []) {
|
||||
if (licenseText.includes(rule.match.toLowerCase())) {
|
||||
return {
|
||||
status: rule.status,
|
||||
rule: rule.match,
|
||||
redistribution: rule.redistribution,
|
||||
commercial_use: rule.commercial_use,
|
||||
share_alike: rule.share_alike,
|
||||
notes: rule.notes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const rule of licenseRegistry.review ?? []) {
|
||||
if (licenseText.includes(rule.match.toLowerCase())) {
|
||||
return { status: rule.status, rule: rule.match, notes: rule.notes }
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: 'review',
|
||||
rule: null,
|
||||
notes: 'No approval rule matched the module license metadata.',
|
||||
}
|
||||
}
|
||||
|
||||
export function swordPackageUrl(moduleId, packageBaseUrl = defaultPackageBaseUrl) {
|
||||
return `${packageBaseUrl.replace(/\/$/, '')}/${moduleId}.zip`
|
||||
}
|
||||
|
||||
export async function writeSwordAudit(outputs) {
|
||||
await writeJson(path.join(process.cwd(), 'audits', 'sword', 'module-audit.json'), outputs)
|
||||
await mkdir(path.join(process.cwd(), 'docs'), { recursive: true })
|
||||
await writeFile(path.join(process.cwd(), 'docs', 'sword-module-audit.md'), swordAuditMarkdown(outputs), 'utf8')
|
||||
}
|
||||
|
||||
export function parseSwordConf(input) {
|
||||
const conf = {}
|
||||
let moduleId = null
|
||||
let currentKey = null
|
||||
|
||||
for (const rawLine of input.split(/\r?\n/)) {
|
||||
const line = rawLine.trim()
|
||||
if (!line || line.startsWith('#')) continue
|
||||
|
||||
const section = /^\[([^\]]+)\]$/.exec(line)
|
||||
if (section) {
|
||||
moduleId = section[1]
|
||||
conf.module_id = moduleId
|
||||
currentKey = null
|
||||
continue
|
||||
}
|
||||
|
||||
const kv = /^([^=]+)=(.*)$/.exec(line)
|
||||
if (kv) {
|
||||
const key = kv[1].trim()
|
||||
const value = cleanSwordValue(kv[2].trim())
|
||||
if (conf[key] === undefined) {
|
||||
conf[key] = value
|
||||
} else if (Array.isArray(conf[key])) {
|
||||
conf[key].push(value)
|
||||
} else {
|
||||
conf[key] = [conf[key], value]
|
||||
}
|
||||
currentKey = key
|
||||
continue
|
||||
}
|
||||
|
||||
if (currentKey && typeof conf[currentKey] === 'string') {
|
||||
conf[currentKey] = `${conf[currentKey]}\n${cleanSwordValue(line)}`
|
||||
}
|
||||
}
|
||||
|
||||
return conf
|
||||
}
|
||||
|
||||
function cleanSwordValue(value) {
|
||||
return value.replace(/\\par\s*/g, '\n').replace(/\s+\n/g, '\n').trim()
|
||||
}
|
||||
|
||||
function readTarEntries(buffer) {
|
||||
const entries = []
|
||||
let offset = 0
|
||||
|
||||
while (offset + 512 <= buffer.length) {
|
||||
const header = buffer.subarray(offset, offset + 512)
|
||||
if (header.every((byte) => byte === 0)) break
|
||||
|
||||
const name = readNullTerminated(header.subarray(0, 100))
|
||||
const sizeText = readNullTerminated(header.subarray(124, 136)).trim()
|
||||
const size = Number.parseInt(sizeText || '0', 8)
|
||||
const contentStart = offset + 512
|
||||
const contentEnd = contentStart + size
|
||||
entries.push({
|
||||
name,
|
||||
size,
|
||||
content: buffer.subarray(contentStart, contentEnd),
|
||||
})
|
||||
|
||||
offset = contentStart + Math.ceil(size / 512) * 512
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
function readNullTerminated(buffer) {
|
||||
const zero = buffer.indexOf(0)
|
||||
return buffer.subarray(0, zero === -1 ? buffer.length : zero).toString('utf8')
|
||||
}
|
||||
|
||||
function swordAuditMarkdown(audit) {
|
||||
const lines = [
|
||||
'# CrossWire SWORD Module Audit',
|
||||
'',
|
||||
'This file is generated from the CrossWire SWORD module catalog and the local LibreBible license approval registry.',
|
||||
'',
|
||||
`- Repository: ${audit.repository.name}`,
|
||||
`- Catalog URL: ${audit.repository.mods_tarball_url}`,
|
||||
`- Catalog SHA-256: \`${audit.catalog_sha256}\``,
|
||||
`- Generated: ${audit.generated_at}`,
|
||||
'',
|
||||
'| Module | Intended Type | SWORD Type | Source Type | License | License Status | Import Status | Package | Notes |',
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- | --- |',
|
||||
]
|
||||
|
||||
for (const module of audit.modules) {
|
||||
lines.push([
|
||||
module.id,
|
||||
module.intended_resource_type,
|
||||
module.mod_drv ?? 'n/a',
|
||||
module.source_type ?? 'n/a',
|
||||
module.distribution_license ?? 'n/a',
|
||||
module.license_status,
|
||||
module.import_status ?? (module.import_enabled === false ? 'deferred' : 'candidate'),
|
||||
module.package_url ? `[zip](${module.package_url})` : 'n/a',
|
||||
module.review_notes.replace(/\|/g, '/'),
|
||||
].join(' | ').replace(/^/, '| ').replace(/$/, ' |'))
|
||||
}
|
||||
|
||||
lines.push('')
|
||||
lines.push('## Next Import Gate')
|
||||
lines.push('')
|
||||
lines.push('A SWORD module should not move from this audit into `packages/json/` until its license metadata is approved and the importer can convert the module content into LibreBible package formats with checksums.')
|
||||
|
||||
return `${lines.join('\n')}\n`
|
||||
}
|
||||
Reference in New Issue
Block a user