Complete storage and search foundation

2026-07-15 16:53:34 -05:00
parent 05ff4bcf9f
commit 2ece7bc36a
12 changed files with 288 additions and 27 deletions
@@ -38,6 +38,8 @@ type ImportedVerse = {

 type ImportedStrongLink = {
  verse_id: string
+  book_id?: string
+  chapter?: number
  position: number
  surface: string
  strong: string
@@ -94,6 +96,12 @@ type ImportedStrongEntry = {
  kjv_definition?: string
 }

+type ImportedStrongIndexEntry = {
+  strong: string
+  verse_ids: string[]
+  surfaces: string[]
+}
+
 type ImportedCommentaryEntry = {
  id: string
  resource_id: string
@@ -168,6 +176,9 @@ type BrowserStrongData = {
 let browserCatalogCache: Promise<ImportedCatalog> | null = null
 const browserLibraryCache = new Map<string, Promise<Verse[]>>()
 const browserStrongDataCache = new Map<string, Promise<BrowserStrongData>>()
+const browserChapterStrongDataCache = new Map<string, Promise<BrowserStrongData>>()
+const browserStrongSearchIndexCache = new Map<string, Promise<Map<string, string[]>>>()
+let browserLexiconCache: Promise<Map<string, BrowserLexiconEntry>> | null = null
 const browserFootnotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
 const browserStudyNotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
 const browserChapterFootnotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
@@ -179,6 +190,7 @@ let browserVersePlacesCache: Promise<Map<string, BiblePlace[]>> | null = null
 let browserRoutesCache: Promise<BibleRoute[]> | null = null
 let browserAudioPlaylistCache: Promise<Map<string, ImportedAudioChapterPlaylist>> | null = null
 const browserVerseSourceIds = new Map<number, string>()
+const browserVerseIdsBySource = new Map<string, number>()
 const browserAudioZipCache = new Map<string, Promise<JSZip>>()
 const browserAudioObjectUrlCache = new Map<string, Promise<string>>()

@@ -329,20 +341,48 @@ async function loadBrowserVersesForTranslation(translationId = defaultTranslatio
  }

  const verses = await browserLibraryCache.get(translationId)!
-  const strongData = await loadBrowserStrongData(translationId).catch((error) => {
-    console.warn('Unable to load browser Strong data.', error)
-    return null
-  })
  const tagsByVerse = readBrowserTags()
  return verses.map((verse) => ({
    ...verse,
-    strongs: strongData?.linksByVerse.get(browserVerseKey(verse)) ?? verse.strongs,
    footnotes: verse.footnotes ?? [],
    studyNotes: verse.studyNotes ?? [],
    tags: tagsByVerse[verse.id] ?? [],
  }))
 }

+async function hydrateBrowserVerses(translationId: string, verses: Verse[]) {
+  const chapterKeys = [
+    ...new Set(
+      verses
+        .map((verse) => `${bookIdFromVerse(verse)}.${verse.chapter}`)
+        .filter((key) => !key.startsWith('unknown.')),
+    ),
+  ]
+  const strongDataMaps = await Promise.all(
+    chapterKeys.map((chapterKey) =>
+      loadBrowserChapterStrongData(translationId, chapterKey).catch((error) => {
+        console.warn(`Unable to load browser Strong data for ${chapterKey}.`, error)
+        return { linksByVerse: new Map<string, StrongLink[]>() }
+      }),
+    ),
+  )
+  const tagsByVerse = readBrowserTags()
+
+  return verses.map((verse) => {
+    const verseKey = browserVerseKey(verse)
+    const strongs =
+      strongDataMaps.find((data) => data.linksByVerse.has(verseKey))?.linksByVerse.get(verseKey) ??
+      verse.strongs
+    return {
+      ...verse,
+      strongs,
+      footnotes: verse.footnotes ?? [],
+      studyNotes: verse.studyNotes ?? [],
+      tags: tagsByVerse[verse.id] ?? [],
+    }
+  })
+}
+
 function allBrowserTags() {
  const unique = new Map<string, Tag>()
  for (const tags of Object.values(readBrowserTags())) {
@@ -375,6 +415,23 @@ async function getBrowserLibraryState(translationId = defaultTranslationId): Pro
  }
 }

+async function getBrowserPageVerses(
+  translationId: string,
+  pageStartId: number,
+  pageSize: number,
+): Promise<Verse[]> {
+  const verses = await loadBrowserVersesForTranslation(translationId)
+  const startIndex = verses.findIndex((verse) => verse.id === pageStartId)
+  if (startIndex < 0) return []
+
+  const book = verses[startIndex].book
+  const pageVerses = verses
+    .slice(startIndex, startIndex + pageSize)
+    .filter((verse) => verse.book === book)
+
+  return hydrateBrowserVerses(translationId, pageVerses)
+}
+
 async function getBrowserTranslationNotesForVerses(
  translationId: string,
  verses: Verse[],
@@ -407,6 +464,14 @@ async function getBrowserTranslationNotesForVerses(
 }

 async function searchBrowserVerses(query: string, translationId = defaultTranslationId): Promise<Verse[]> {
+  if (/^[gh]\d+$/i.test(query.trim())) {
+    const verses = await loadBrowserVersesForTranslation(translationId)
+    const strongIndex = await loadBrowserStrongSearchIndex(translationId)
+    const sourceIds = strongIndex.get(query.trim().toUpperCase()) ?? []
+    const verseIds = new Set(sourceIds.map((sourceId) => browserVerseIdsBySource.get(sourceId)))
+    return verses.filter((verse) => verseIds.has(verse.id))
+  }
+
  const searchIndex = await loadBrowserSearchIndex(translationId)
  return searchIndex.search(query)
 }
@@ -552,6 +617,7 @@ const emptyMapContext: MapContext = {
 const browserStudyDataClient: StudyDataClient = {
  getTranslations: getBrowserTranslations,
  getLibraryState: getBrowserLibraryState,
+  getPageVerses: getBrowserPageVerses,
  getTranslationNotesForVerses: getBrowserTranslationNotesForVerses,
  searchVerses: searchBrowserVerses,
  addTagToVerse: addBrowserTagToVerse,
@@ -565,6 +631,13 @@ const browserStudyDataClient: StudyDataClient = {
 const tauriStudyDataClient: StudyDataClient = {
  getTranslations: async () => [],
  getLibraryState: () => invoke<LibraryState>('get_library_state'),
+  getPageVerses: async (_translationId, pageStartId, pageSize) => {
+    const state = await invoke<LibraryState>('get_library_state')
+    const startIndex = state.verses.findIndex((verse) => verse.id === pageStartId)
+    if (startIndex < 0) return []
+    const book = state.verses[startIndex].book
+    return state.verses.slice(startIndex, startIndex + pageSize).filter((verse) => verse.book === book)
+  },
  getTranslationNotesForVerses: async () => new Map(),
  searchVerses: (query) => invoke<Verse[]>('search_verses', { query }),
  addTagToVerse: (verseId, label, color) => invoke<Verse>('add_tag_to_verse', { verseId, label, color }),
@@ -589,6 +662,14 @@ export function getLibraryState(translationId = defaultTranslationId): Promise<L
  return getStudyDataClient().getLibraryState(translationId)
 }

+export function getPageVerses(
+  translationId: string,
+  pageStartId: number,
+  pageSize: number,
+): Promise<Verse[]> {
+  return getStudyDataClient().getPageVerses(translationId, pageStartId, pageSize)
+}
+
 export function getTranslationNotesForVerses(
  translationId: string,
  verses: Verse[],
@@ -648,15 +729,42 @@ async function loadBrowserSearchIndex(translationId = defaultTranslationId) {
  return browserSearchIndexCache.get(translationId)!
 }

+async function loadBrowserStrongSearchIndex(translationId = defaultTranslationId) {
+  if (!browserStrongSearchIndexCache.has(translationId)) {
+    browserStrongSearchIndexCache.set(
+      translationId,
+      fetchJsonl<ImportedStrongIndexEntry>(`${libreBibleDevRoot}/${translationId}/strongs-index.jsonl`)
+        .then((entries) => new Map(entries.map((entry) => [entry.strong.toUpperCase(), entry.verse_ids])))
+        .catch(async () => {
+          const strongData = await loadBrowserStrongData(translationId)
+          const index = new Map<string, string[]>()
+          for (const [verseId, links] of strongData.linksByVerse) {
+            for (const link of links) {
+              const key = link.strong.toUpperCase()
+              const current = index.get(key) ?? []
+              if (!current.includes(verseId)) current.push(verseId)
+              index.set(key, current)
+            }
+          }
+          return index
+        }),
+    )
+  }
+
+  return browserStrongSearchIndexCache.get(translationId)!
+}
+
 async function loadLibreBibleDevVerses(translationId = defaultTranslationId): Promise<Verse[]> {
  const importedVerses = await fetchJsonl<ImportedVerse>(
    `${libreBibleDevRoot}/${translationId}/verses.jsonl`,
  )

  browserVerseSourceIds.clear()
+  browserVerseIdsBySource.clear()
  return importedVerses.map((verse, index) => {
    const id = index + 1
    browserVerseSourceIds.set(id, verse.id)
+    browserVerseIdsBySource.set(verse.id, id)
    return {
      id,
      book: verse.book,
@@ -982,6 +1090,42 @@ function mapImportedNotes(importedFootnotes: ImportedFootnote[]): Map<string, Fo
  return footnotes
 }

+async function loadBrowserChapterStrongData(
+  translationId: string,
+  chapterKey: string,
+): Promise<BrowserStrongData> {
+  const cacheKey = `${translationId}:${chapterKey}`
+  if (!browserChapterStrongDataCache.has(cacheKey)) {
+    browserChapterStrongDataCache.set(
+      cacheKey,
+      loadLibreBibleDevChapterStrongData(translationId, chapterKey).catch(async () => {
+        const fullStrongData = await loadBrowserStrongData(translationId)
+        const linksByVerse = new Map<string, StrongLink[]>()
+        for (const [verseId, links] of fullStrongData.linksByVerse) {
+          if (verseId.startsWith(`${chapterKey}.`)) linksByVerse.set(verseId, links)
+        }
+        return { linksByVerse }
+      }),
+    )
+  }
+
+  return browserChapterStrongDataCache.get(cacheKey)!
+}
+
+async function loadLibreBibleDevChapterStrongData(
+  translationId: string,
+  chapterKey: string,
+): Promise<BrowserStrongData> {
+  const [importedLinks, lexicon] = await Promise.all([
+    fetchJsonl<ImportedStrongLink>(
+      `${libreBibleDevRoot}/${translationId}/strongs-by-chapter/${chapterKey}.strongs.jsonl`,
+    ),
+    loadBrowserLexicon(),
+  ])
+
+  return mapStrongLinks(importedLinks, lexicon)
+}
+
 async function loadBrowserStrongData(translationId = defaultTranslationId): Promise<BrowserStrongData> {
  if (!browserStrongDataCache.has(translationId)) {
    browserStrongDataCache.set(translationId, loadLibreBibleDevStrongData(translationId))
@@ -996,7 +1140,19 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId)
    fetchJsonl<ImportedStrongEntry>(`${libreBibleDevRoot}/strongs-open-scriptures/entries.jsonl`),
  ])

-  const lexicon = new Map<string, BrowserLexiconEntry>(
+  return mapStrongLinks(importedLinks, mapBrowserLexicon(importedEntries))
+}
+
+async function loadBrowserLexicon(): Promise<Map<string, BrowserLexiconEntry>> {
+  browserLexiconCache ??= fetchJsonl<ImportedStrongEntry>(
+    `${libreBibleDevRoot}/strongs-open-scriptures/entries.jsonl`,
+  ).then(mapBrowserLexicon)
+
+  return browserLexiconCache
+}
+
+function mapBrowserLexicon(importedEntries: ImportedStrongEntry[]) {
+  return new Map<string, BrowserLexiconEntry>(
    importedEntries.map((entry) => [
      entry.strongs_number,
      {
@@ -1007,6 +1163,12 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId)
      },
    ]),
  )
+}
+
+function mapStrongLinks(
+  importedLinks: ImportedStrongLink[],
+  lexicon: Map<string, BrowserLexiconEntry>,
+): BrowserStrongData {
  const linksByVerse = new Map<string, StrongLink[]>()

  for (const link of importedLinks) {