libre-bible-data/scripts/sword-lib.js

import { createHash } from 'node:crypto'
import { gunzipSync } from 'node:zlib'
import { mkdir, readFile, writeFile } from 'node:fs/promises'
import path from 'node:path'
import { exists, readJson, writeJson } from './lib.js'

export const swordCacheDir = path.join(process.cwd(), 'cache', 'sword')
export const defaultModsTarballUrl = 'https://crosswire.org/ftpmirror/pub/sword/raw/mods.d.tar.gz'
export const defaultPackageBaseUrl = 'https://crosswire.org/ftpmirror/pub/sword/packages/rawzip'

export async function fetchSwordConfCatalog(url = defaultModsTarballUrl) {
  await mkdir(swordCacheDir, { recursive: true })
  const tarballPath = path.join(swordCacheDir, 'mods.d.tar.gz')

  if (!(await exists(tarballPath))) {
    const response = await fetch(url)
    if (!response.ok) {
      throw new Error(`Unable to fetch SWORD module catalog ${url}: ${response.status} ${response.statusText}`)
    }
    await writeFile(tarballPath, Buffer.from(await response.arrayBuffer()))
  }

  const tarball = await readFile(tarballPath)
  const sha256 = createHash('sha256').update(tarball).digest('hex')
  const tar = gunzipSync(tarball)
  const modules = new Map()

  for (const entry of readTarEntries(tar)) {
    if (!entry.name.endsWith('.conf')) continue
    const text = entry.content.toString('utf8')
    const conf = parseSwordConf(text)
    if (conf.module_id) {
      modules.set(conf.module_id.toLowerCase(), {
        module_id: conf.module_id,
        conf_file: entry.name,
        conf_text: text,
        conf,
      })
    }
  }

  return {
    source_url: url,
    tarball_path: tarballPath,
    sha256,
    modules,
  }
}

export async function loadSwordPilotModules(filePath = path.join(process.cwd(), 'registries', 'sword-pilot-modules.json')) {
  return readJson(filePath)
}

export async function loadLicenseApproval(filePath = path.join(process.cwd(), 'registries', 'license-approval.json')) {
  return readJson(filePath)
}

export function classifySwordModule(conf, licenseRegistry) {
  const licenseText = [
    conf.DistributionLicense,
    conf.Copyright,
    conf.About,
    conf.ShortPromo,
  ].filter(Boolean).join('\n').toLowerCase()

  for (const rule of licenseRegistry.rejected ?? []) {
    if (licenseText.includes(rule.match.toLowerCase())) {
      return { status: rule.status, rule: rule.match, notes: rule.notes }
    }
  }

  for (const rule of licenseRegistry.approved ?? []) {
    if (licenseText.includes(rule.match.toLowerCase())) {
      return {
        status: rule.status,
        rule: rule.match,
        redistribution: rule.redistribution,
        commercial_use: rule.commercial_use,
        share_alike: rule.share_alike,
        notes: rule.notes,
      }
    }
  }

  for (const rule of licenseRegistry.review ?? []) {
    if (licenseText.includes(rule.match.toLowerCase())) {
      return { status: rule.status, rule: rule.match, notes: rule.notes }
    }
  }

  return {
    status: 'review',
    rule: null,
    notes: 'No approval rule matched the module license metadata.',
  }
}

export function swordPackageUrl(moduleId, packageBaseUrl = defaultPackageBaseUrl) {
  return `${packageBaseUrl.replace(/\/$/, '')}/${moduleId}.zip`
}

export async function writeSwordAudit(outputs) {
  await writeJson(path.join(process.cwd(), 'audits', 'sword', 'module-audit.json'), outputs)
  await mkdir(path.join(process.cwd(), 'docs'), { recursive: true })
  await writeFile(path.join(process.cwd(), 'docs', 'sword-module-audit.md'), swordAuditMarkdown(outputs), 'utf8')
}

export function parseSwordConf(input) {
  const conf = {}
  let moduleId = null
  let currentKey = null

  for (const rawLine of input.split(/\r?\n/)) {
    const line = rawLine.trim()
    if (!line || line.startsWith('#')) continue

    const section = /^\[([^\]]+)\]$/.exec(line)
    if (section) {
      moduleId = section[1]
      conf.module_id = moduleId
      currentKey = null
      continue
    }

    const kv = /^([^=]+)=(.*)$/.exec(line)
    if (kv) {
      const key = kv[1].trim()
      const value = cleanSwordValue(kv[2].trim())
      if (conf[key] === undefined) {
        conf[key] = value
      } else if (Array.isArray(conf[key])) {
        conf[key].push(value)
      } else {
        conf[key] = [conf[key], value]
      }
      currentKey = key
      continue
    }

    if (currentKey && typeof conf[currentKey] === 'string') {
      conf[currentKey] = `${conf[currentKey]}\n${cleanSwordValue(line)}`
    }
  }

  return conf
}

function cleanSwordValue(value) {
  return value.replace(/\\par\s*/g, '\n').replace(/\s+\n/g, '\n').trim()
}

function readTarEntries(buffer) {
  const entries = []
  let offset = 0

  while (offset + 512 <= buffer.length) {
    const header = buffer.subarray(offset, offset + 512)
    if (header.every((byte) => byte === 0)) break

    const name = readNullTerminated(header.subarray(0, 100))
    const sizeText = readNullTerminated(header.subarray(124, 136)).trim()
    const size = Number.parseInt(sizeText || '0', 8)
    const contentStart = offset + 512
    const contentEnd = contentStart + size
    entries.push({
      name,
      size,
      content: buffer.subarray(contentStart, contentEnd),
    })

    offset = contentStart + Math.ceil(size / 512) * 512
  }

  return entries
}

function readNullTerminated(buffer) {
  const zero = buffer.indexOf(0)
  return buffer.subarray(0, zero === -1 ? buffer.length : zero).toString('utf8')
}

function swordAuditMarkdown(audit) {
  const lines = [
    '# CrossWire SWORD Module Audit',
    '',
    'This file is generated from the CrossWire SWORD module catalog and the local LibreBible license approval registry.',
    '',
    `- Repository: ${audit.repository.name}`,
    `- Catalog URL: ${audit.repository.mods_tarball_url}`,
    `- Catalog SHA-256: \`${audit.catalog_sha256}\``,
    `- Generated: ${audit.generated_at}`,
    '',
    '| Module | Intended Type | SWORD Type | Source Type | License | License Status | Import Status | Package | Notes |',
    '| --- | --- | --- | --- | --- | --- | --- | --- | --- |',
  ]

  for (const module of audit.modules) {
    lines.push([
      module.id,
      module.intended_resource_type,
      module.mod_drv ?? 'n/a',
      module.source_type ?? 'n/a',
      module.distribution_license ?? 'n/a',
      module.license_status,
      module.import_status ?? (module.import_enabled === false ? 'deferred' : 'candidate'),
      module.package_url ? `[zip](${module.package_url})` : 'n/a',
      module.review_notes.replace(/\|/g, '/'),
    ].join(' | ').replace(/^/, '| ').replace(/$/, ' |'))
  }

  lines.push('')
  lines.push('## Next Import Gate')
  lines.push('')
  lines.push('A SWORD module should not move from this audit into `packages/json/` until its license metadata is approved and the importer can convert the module content into LibreBible package formats with checksums.')

  return `${lines.join('\n')}\n`
}