Add storage search foundation

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.1.26 - 2026-07-12
+
+- Started Phase 5 storage/search foundation with a reusable verse search index for browser/local preview queries.
+- Routed browser verse search through the cached query layer instead of open-coded filtering.
+- Added storage/search foundation documentation covering resource/user data boundaries, query goals, SQLite tables, indexes, and future full-text search.
+
 ## 0.1.25 - 2026-07-12
 
 - Started Phase 4 shared app core cleanup by extracting workspace/session/settings/timeline schema and persistence adapters into `src/workspace.ts`.

ROADMAP.md

@@ -127,6 +127,8 @@ Definition of done:
 
 Goal: make hundreds of thousands or millions of entries feel normal.
 
+Current foundation note: [Storage And Search Foundation](docs/storage-search-foundation.md).
+
 Tasks:
 
 - Keep SQLite for desktop and private/single-user workflows.
@@ -135,6 +137,7 @@ Tasks:
 - Add proper indexes for translation id, book/chapter/verse, Strong's number, lemma, resource id, place id, and attachment relationships.
 - Add full-text search through SQLite FTS5 or a compatible indexed search layer.
 - Avoid loading entire large resources into the browser when a chapter, page, verse, Strong's entry, or search result can be loaded on demand.
+- Keep the UI pointed at shared query/search interfaces so JSON packages, Tauri SQLite, hosted SQLite, and future server search can be swapped without rewriting the reader.
 
 Definition of done:
 

docs/storage-search-foundation.md

@@ -0,0 +1,91 @@
+# Storage And Search Foundation
+
+Libre Study should query study resources through stable app-level search and resource interfaces. The UI should not care whether a result came from local JSON packages, browser indexes, Tauri SQLite, hosted SQLite, or a future server database.
+
+## Current Phase 5 Step
+
+- Browser/local preview still loads LibreBible JSON/JSONL packages.
+- Verse search now goes through a cached verse search index instead of open-coded filtering.
+- The index supports normal text search, reference text, `tag:` search, and Strong's-number search.
+- The index is disposable and rebuildable from the loaded translation package, which keeps it compatible with later SQLite/server-backed search.
+
+## Resource Data vs User Data
+
+Public resource data belongs to LibreBible:
+
+- translations
+- Strong's links
+- lexicons and dictionaries
+- footnotes and study notes
+- commentaries
+- maps and place metadata
+- audio Bible metadata
+- pronunciation metadata
+
+User workspace data belongs to Libre Study:
+
+- sessions
+- notes
+- visual settings
+- timeline events
+- tags
+- highlights
+- saved searches
+- future collaboration state
+
+## Query Interface Goals
+
+The app should ask stable questions:
+
+- search this translation for text
+- find verses by Strong's number
+- get notes for visible verses
+- get commentary for a verse or range
+- get dictionary/lexicon entries for terms, lemmas, or Strong's numbers
+- get maps for places connected to this verse/page
+- get audio for this chapter, verse, or lemma
+
+The backing implementation can change without rewriting the page UI.
+
+## SQLite Index Plan
+
+SQLite remains appropriate for desktop, local, and private alpha use. It needs explicit tables and indexes instead of ad hoc file reads.
+
+Core resource tables:
+
+- `resources`: resource id, type, title, version, license, package version.
+- `verses`: translation id, source verse id, book, chapter, verse, text.
+- `verse_strongs`: translation id, source verse id, position, surface, Strong's number.
+- `lexicon_entries`: Strong's number, language, lemma, transliteration, pronunciation, definitions.
+- `verse_notes`: resource id, translation id, source verse id, note type, caller, text.
+- `commentary_entries`: resource id, book, chapter, verse start, verse end, title, text.
+- `dictionary_entries`: resource id, term, slug, definitions, scripture refs.
+- `places`: place id, canonical name, aliases, coordinates, bounds.
+- `verse_places`: source verse id or range, place id, confidence/source.
+- `audio_segments`: resource id, translation/language, book, chapter, verse range, file path/url, timestamps.
+- `pronunciations`: resource id, language, term, lemma, Strong's number, file path/url, source.
+
+Indexes:
+
+- `verses(translation_id, book, chapter, verse)`
+- `verses(source_verse_id)`
+- `verse_strongs(strong)`
+- `verse_strongs(translation_id, source_verse_id)`
+- `verse_notes(translation_id, source_verse_id, note_type)`
+- `commentary_entries(book, chapter, verse_start, verse_end)`
+- `dictionary_entries(term)`
+- `places(canonical_name)`
+- `verse_places(place_id)`
+- `audio_segments(book, chapter, verse_start, verse_end)`
+- `pronunciations(strong_number)`
+- `pronunciations(lemma)`
+
+Full-text indexes:
+
+- Bible text through SQLite FTS5.
+- Notes and commentaries through SQLite FTS5.
+- Dictionary/lexicon definitions through SQLite FTS5.
+
+## Hosted Web Path
+
+For the private alpha, server-side SQLite is acceptable. Before public multi-user accounts and collaboration, user/account/session data should move to a server database such as Postgres while public resource packages can remain static, indexed, or separately cached.

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "libre-study",
-  "version": "0.1.25",
+  "version": "0.1.26",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "libre-study",
-      "version": "0.1.25",
+      "version": "0.1.26",
       "dependencies": {
         "@tauri-apps/api": "^2.11.1",
         "react": "^19.2.7",

package.json

@@ -1,7 +1,7 @@
 {
   "name": "libre-study",
   "private": true,
-  "version": "0.1.25",
+  "version": "0.1.26",
   "type": "module",
   "scripts": {
     "dev": "vite --host 127.0.0.1",

src-tauri/Cargo.lock

@@ -77,7 +77,7 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
 
 [[package]]
 name = "app"
-version = "0.1.25"
+version = "0.1.26"
 dependencies = [
  "log",
  "rusqlite",

src-tauri/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "app"
-version = "0.1.25"
+version = "0.1.26"
 description = "A Tauri App"
 authors = ["you"]
 license = ""

src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
   "productName": "Libre Study",
-  "version": "0.1.25",
+  "version": "0.1.26",
   "identifier": "org.librestudy.desktop",
   "build": {
     "frontendDist": "../dist",

src/searchIndex.ts

@@ -0,0 +1,116 @@
+import type { Verse } from './types'
+
+export type VerseSearchIndex = {
+  search(query: string): Verse[]
+}
+
+const strongPattern = /^[gh]\d+$/i
+
+export function buildVerseSearchIndex(verses: Verse[]): VerseSearchIndex {
+  const indexedVerses = verses.map((verse) => ({
+    verse,
+    haystack: normalizeText(`${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`),
+  }))
+  const versesById = new Map(verses.map((verse) => [verse.id, verse]))
+  const tokens = new Map<string, Set<number>>()
+  const strongs = new Map<string, Set<number>>()
+  const tags = new Map<string, Set<number>>()
+
+  for (const verse of verses) {
+    addTokens(tokens, verse.id, `${verse.book} ${verse.chapter} ${verse.verse} ${verse.text}`)
+    for (const strong of verse.strongs) {
+      addToIndex(strongs, strong.strong.toUpperCase(), verse.id)
+      addTokens(tokens, verse.id, strong.surface)
+      addTokens(tokens, verse.id, strong.lemma)
+      addTokens(tokens, verse.id, strong.transliteration)
+    }
+    for (const tag of verse.tags) {
+      addToIndex(tags, normalizeTag(tag.label), verse.id)
+    }
+  }
+
+  return {
+    search(query: string) {
+      const trimmed = query.trim()
+      if (!trimmed) return verses
+
+      const lowered = trimmed.toLowerCase()
+      if (lowered.startsWith('tag:')) {
+        return verseIdsToVerses(tags.get(normalizeTag(trimmed.slice(4))) ?? new Set(), versesById)
+      }
+
+      if (strongPattern.test(trimmed)) {
+        return verseIdsToVerses(strongs.get(trimmed.toUpperCase()) ?? new Set(), versesById)
+      }
+
+      const normalizedQuery = normalizeText(trimmed)
+      const queryTokens = tokenize(normalizedQuery)
+      if (queryTokens.length) {
+        const candidateIds = intersectTokenMatches(tokens, queryTokens)
+        if (candidateIds) {
+          const candidates = verseIdsToVerses(candidateIds, versesById)
+          if (queryTokens.length === 1) return candidates
+          return candidates.filter((verse) =>
+            normalizeText(`${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`).includes(normalizedQuery),
+          )
+        }
+      }
+
+      return indexedVerses
+        .filter((entry) => entry.haystack.includes(normalizedQuery))
+        .map((entry) => entry.verse)
+    },
+  }
+}
+
+function addTokens(index: Map<string, Set<number>>, verseId: number, value: string) {
+  for (const token of tokenize(normalizeText(value))) {
+    addToIndex(index, token, verseId)
+  }
+}
+
+function addToIndex(index: Map<string, Set<number>>, key: string, verseId: number) {
+  if (!key) return
+  const current = index.get(key) ?? new Set<number>()
+  current.add(verseId)
+  index.set(key, current)
+}
+
+function intersectTokenMatches(index: Map<string, Set<number>>, queryTokens: string[]) {
+  let match: Set<number> | null = null
+
+  for (const token of queryTokens) {
+    const tokenMatches = index.get(token)
+    if (!tokenMatches) return null
+    if (!match) {
+      match = new Set(tokenMatches)
+      continue
+    }
+
+    match = new Set([...match].filter((id) => tokenMatches.has(id)))
+    if (!match.size) return match
+  }
+
+  return match
+}
+
+function verseIdsToVerses(ids: Set<number>, versesById: Map<number, Verse>) {
+  const verses: Verse[] = []
+  for (const id of ids) {
+    const verse = versesById.get(id)
+    if (verse) verses.push(verse)
+  }
+  return verses
+}
+
+function tokenize(value: string) {
+  return value.match(/[a-z0-9]+/g) ?? []
+}
+
+function normalizeTag(value: string) {
+  return value.trim().toLowerCase()
+}
+
+function normalizeText(value: string) {
+  return value.toLowerCase().replace(/[^a-z0-9\s:]/g, ' ').replace(/\s+/g, ' ').trim()
+}

src/studyClient.ts

@@ -1,4 +1,5 @@
 import { invoke } from '@tauri-apps/api/core'
+import { buildVerseSearchIndex } from './searchIndex'
 import type {
   CommentaryEntry,
   DictionaryEntry,
@@ -110,7 +111,6 @@ type BrowserLexiconEntry = {
 
 type BrowserStrongData = {
   linksByVerse: Map<string, StrongLink[]>
-  verseIdsByStrong: Map<string, Set<string>>
 }
 
 let browserCatalogCache: Promise<ImportedCatalog> | null = null
@@ -120,6 +120,7 @@ const browserFootnotesCache = new Map<string, Promise<Map<string, Footnote[]>>>(
 const browserStudyNotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
 const browserChapterFootnotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
 const browserChapterStudyNotesCache = new Map<string, Promise<Map<string, Footnote[]>>>()
+const browserSearchIndexCache = new Map<string, Promise<ReturnType<typeof buildVerseSearchIndex>>>()
 let browserCommentaryCache: Promise<Map<string, CommentaryEntry[]>> | null = null
 let browserDictionaryCache: Promise<Map<string, DictionaryEntry>> | null = null
 const browserVerseSourceIds = new Map<number, string>()
@@ -359,30 +360,8 @@ export async function searchVerses(query: string, translationId = defaultTransla
     return invoke<Verse[]>('search_verses', { query })
   }
 
-  const trimmed = query.trim().toLowerCase()
-  const verses = await loadBrowserVersesForTranslation(translationId)
-  if (!trimmed) return verses
-
-  if (trimmed.startsWith('tag:')) {
-    const label = trimmed.slice(4)
-    return verses.filter((verse) =>
-      verse.tags.some((tag) => tag.label.toLowerCase() === label),
-    )
-  }
-
-  if (/^[gh]\d+$/.test(trimmed)) {
-    const strongData = await loadBrowserStrongData(translationId)
-    const matchingVerseIds = strongData.verseIdsByStrong.get(trimmed.toUpperCase()) ?? new Set()
-    return verses.filter((verse) =>
-      matchingVerseIds.has(browserVerseKey(verse)),
-    )
-  }
-
-  return verses.filter((verse) =>
-    `${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`
-      .toLowerCase()
-      .includes(trimmed),
-  )
+  const searchIndex = await loadBrowserSearchIndex(translationId)
+  return searchIndex.search(query)
 }
 
 export async function addTagToVerse(
@@ -401,6 +380,7 @@ export async function addTagToVerse(
   const tag = existing ?? { id: Date.now(), label, color }
   tagsByVerse[verseId] = existing ? current : [...current, tag]
   writeBrowserTags(tagsByVerse)
+  browserSearchIndexCache.delete(translationId)
 
   const verse = (await loadBrowserVersesForTranslation(translationId)).find((item) => item.id === verseId)
   if (!verse) throw new Error(`Verse ${verseId} was not found.`)
@@ -441,6 +421,17 @@ async function loadBrowserCatalog(): Promise<ImportedCatalog> {
   return browserCatalogCache
 }
 
+async function loadBrowserSearchIndex(translationId = defaultTranslationId) {
+  if (!browserSearchIndexCache.has(translationId)) {
+    browserSearchIndexCache.set(
+      translationId,
+      loadBrowserVersesForTranslation(translationId).then((verses) => buildVerseSearchIndex(verses)),
+    )
+  }
+
+  return browserSearchIndexCache.get(translationId)!
+}
+
 async function loadLibreBibleDevVerses(translationId = defaultTranslationId): Promise<Verse[]> {
   const importedVerses = await fetchJsonl<ImportedVerse>(
     `${libreBibleDevRoot}/${translationId}/verses.jsonl`,
@@ -652,7 +643,6 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId)
     ]),
   )
   const linksByVerse = new Map<string, StrongLink[]>()
-  const verseIdsByStrong = new Map<string, Set<string>>()
 
   for (const link of importedLinks) {
     const entry = lexicon.get(link.strong)
@@ -668,17 +658,13 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId)
     const current = linksByVerse.get(link.verse_id) ?? []
     current.push(strongLink)
     linksByVerse.set(link.verse_id, current)
-
-    const strongMatches = verseIdsByStrong.get(link.strong) ?? new Set<string>()
-    strongMatches.add(link.verse_id)
-    verseIdsByStrong.set(link.strong, strongMatches)
   }
 
   for (const links of linksByVerse.values()) {
     links.sort((a, b) => a.position - b.position)
   }
 
-  return { linksByVerse, verseIdsByStrong }
+  return { linksByVerse }
 }
 
 function strongLanguage(strong: string): 'Hebrew' | 'Greek' {