Add storage search foundation

2026-07-12 13:00:12 -05:00
parent 8aafd3061b
commit a1a902c938
10 changed files with 239 additions and 37 deletions
@@ -0,0 +1,116 @@
+import type { Verse } from './types'
+
+export type VerseSearchIndex = {
+  search(query: string): Verse[]
+}
+
+const strongPattern = /^[gh]\d+$/i
+
+export function buildVerseSearchIndex(verses: Verse[]): VerseSearchIndex {
+  const indexedVerses = verses.map((verse) => ({
+    verse,
+    haystack: normalizeText(`${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`),
+  }))
+  const versesById = new Map(verses.map((verse) => [verse.id, verse]))
+  const tokens = new Map<string, Set<number>>()
+  const strongs = new Map<string, Set<number>>()
+  const tags = new Map<string, Set<number>>()
+
+  for (const verse of verses) {
+    addTokens(tokens, verse.id, `${verse.book} ${verse.chapter} ${verse.verse} ${verse.text}`)
+    for (const strong of verse.strongs) {
+      addToIndex(strongs, strong.strong.toUpperCase(), verse.id)
+      addTokens(tokens, verse.id, strong.surface)
+      addTokens(tokens, verse.id, strong.lemma)
+      addTokens(tokens, verse.id, strong.transliteration)
+    }
+    for (const tag of verse.tags) {
+      addToIndex(tags, normalizeTag(tag.label), verse.id)
+    }
+  }
+
+  return {
+    search(query: string) {
+      const trimmed = query.trim()
+      if (!trimmed) return verses
+
+      const lowered = trimmed.toLowerCase()
+      if (lowered.startsWith('tag:')) {
+        return verseIdsToVerses(tags.get(normalizeTag(trimmed.slice(4))) ?? new Set(), versesById)
+      }
+
+      if (strongPattern.test(trimmed)) {
+        return verseIdsToVerses(strongs.get(trimmed.toUpperCase()) ?? new Set(), versesById)
+      }
+
+      const normalizedQuery = normalizeText(trimmed)
+      const queryTokens = tokenize(normalizedQuery)
+      if (queryTokens.length) {
+        const candidateIds = intersectTokenMatches(tokens, queryTokens)
+        if (candidateIds) {
+          const candidates = verseIdsToVerses(candidateIds, versesById)
+          if (queryTokens.length === 1) return candidates
+          return candidates.filter((verse) =>
+            normalizeText(`${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`).includes(normalizedQuery),
+          )
+        }
+      }
+
+      return indexedVerses
+        .filter((entry) => entry.haystack.includes(normalizedQuery))
+        .map((entry) => entry.verse)
+    },
+  }
+}
+
+function addTokens(index: Map<string, Set<number>>, verseId: number, value: string) {
+  for (const token of tokenize(normalizeText(value))) {
+    addToIndex(index, token, verseId)
+  }
+}
+
+function addToIndex(index: Map<string, Set<number>>, key: string, verseId: number) {
+  if (!key) return
+  const current = index.get(key) ?? new Set<number>()
+  current.add(verseId)
+  index.set(key, current)
+}
+
+function intersectTokenMatches(index: Map<string, Set<number>>, queryTokens: string[]) {
+  let match: Set<number> | null = null
+
+  for (const token of queryTokens) {
+    const tokenMatches = index.get(token)
+    if (!tokenMatches) return null
+    if (!match) {
+      match = new Set(tokenMatches)
+      continue
+    }
+
+    match = new Set([...match].filter((id) => tokenMatches.has(id)))
+    if (!match.size) return match
+  }
+
+  return match
+}
+
+function verseIdsToVerses(ids: Set<number>, versesById: Map<number, Verse>) {
+  const verses: Verse[] = []
+  for (const id of ids) {
+    const verse = versesById.get(id)
+    if (verse) verses.push(verse)
+  }
+  return verses
+}
+
+function tokenize(value: string) {
+  return value.match(/[a-z0-9]+/g) ?? []
+}
+
+function normalizeTag(value: string) {
+  return value.trim().toLowerCase()
+}
+
+function normalizeText(value: string) {
+  return value.toLowerCase().replace(/[^a-z0-9\s:]/g, ' ').replace(/\s+/g, ' ').trim()
+}
@@ -1,4 +1,5 @@
 import { invoke } from '@tauri-apps/api/core'
+import { buildVerseSearchIndex } from './searchIndex'
 import type {
  CommentaryEntry,
  DictionaryEntry,
@@ -110,7 +111,6 @@ type BrowserLexiconEntry = {

 type BrowserStrongData = {
  linksByVerse: Map<string, StrongLink[]>
-  verseIdsByStrong: Map<string, Set<string>>
 }

 let browserCatalogCache: Promise<ImportedCatalog> | null = null
@@ -120,6 +120,7 @@ const browserFootnotesCache = new Map<string, Promise<Map<string, Footnote[]>>>(
 const browserStudyNotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
 const browserChapterFootnotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
 const browserChapterStudyNotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
+const browserSearchIndexCache = new Map<string, Promise<ReturnType<typeof buildVerseSearchIndex>>>()
 let browserCommentaryCache: Promise<Map<string, CommentaryEntry[]>> | null = null
 let browserDictionaryCache: Promise<Map<string, DictionaryEntry>> | null = null
 const browserVerseSourceIds = new Map<number, string>()
@@ -359,30 +360,8 @@ export async function searchVerses(query: string, translationId = defaultTransla
    return invoke<Verse[]>('search_verses', { query })
  }

-  const trimmed = query.trim().toLowerCase()
-  const verses = await loadBrowserVersesForTranslation(translationId)
-  if (!trimmed) return verses
-
-  if (trimmed.startsWith('tag:')) {
-    const label = trimmed.slice(4)
-    return verses.filter((verse) =>
-      verse.tags.some((tag) => tag.label.toLowerCase() === label),
-    )
-  }
-
-  if (/^[gh]\d+$/.test(trimmed)) {
-    const strongData = await loadBrowserStrongData(translationId)
-    const matchingVerseIds = strongData.verseIdsByStrong.get(trimmed.toUpperCase()) ?? new Set()
-    return verses.filter((verse) =>
-      matchingVerseIds.has(browserVerseKey(verse)),
-    )
-  }
-
-  return verses.filter((verse) =>
-    `${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`
-      .toLowerCase()
-      .includes(trimmed),
-  )
+  const searchIndex = await loadBrowserSearchIndex(translationId)
+  return searchIndex.search(query)
 }

 export async function addTagToVerse(
@@ -401,6 +380,7 @@ export async function addTagToVerse(
  const tag = existing ?? { id: Date.now(), label, color }
  tagsByVerse[verseId] = existing ? current : [...current, tag]
  writeBrowserTags(tagsByVerse)
+  browserSearchIndexCache.delete(translationId)

  const verse = (await loadBrowserVersesForTranslation(translationId)).find((item) => item.id === verseId)
  if (!verse) throw new Error(`Verse ${verseId} was not found.`)
@@ -441,6 +421,17 @@ async function loadBrowserCatalog(): Promise<ImportedCatalog> {
  return browserCatalogCache
 }

+async function loadBrowserSearchIndex(translationId = defaultTranslationId) {
+  if (!browserSearchIndexCache.has(translationId)) {
+    browserSearchIndexCache.set(
+      translationId,
+      loadBrowserVersesForTranslation(translationId).then((verses) => buildVerseSearchIndex(verses)),
+    )
+  }
+
+  return browserSearchIndexCache.get(translationId)!
+}
+
 async function loadLibreBibleDevVerses(translationId = defaultTranslationId): Promise<Verse[]> {
  const importedVerses = await fetchJsonl<ImportedVerse>(
    `${libreBibleDevRoot}/${translationId}/verses.jsonl`,
@@ -652,7 +643,6 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId)
    ]),
  )
  const linksByVerse = new Map<string, StrongLink[]>()
-  const verseIdsByStrong = new Map<string, Set<string>>()

  for (const link of importedLinks) {
    const entry = lexicon.get(link.strong)
@@ -668,17 +658,13 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId)
    const current = linksByVerse.get(link.verse_id) ?? []
    current.push(strongLink)
    linksByVerse.set(link.verse_id, current)
-
-    const strongMatches = verseIdsByStrong.get(link.strong) ?? new Set<string>()
-    strongMatches.add(link.verse_id)
-    verseIdsByStrong.set(link.strong, strongMatches)
  }

  for (const links of linksByVerse.values()) {
    links.sort((a, b) => a.position - b.position)
  }

-  return { linksByVerse, verseIdsByStrong }
+  return { linksByVerse }
 }

 function strongLanguage(strong: string): 'Hebrew' | 'Greek' {