From a1a902c938b1e3a27a44f1827f877517e2182546 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 12 Jul 2026 13:00:12 -0500 Subject: [PATCH] Add storage search foundation --- CHANGELOG.md | 6 ++ ROADMAP.md | 3 + docs/storage-search-foundation.md | 91 +++++++++++++++++++++++ package-lock.json | 4 +- package.json | 2 +- src-tauri/Cargo.lock | 2 +- src-tauri/Cargo.toml | 2 +- src-tauri/tauri.conf.json | 2 +- src/searchIndex.ts | 116 ++++++++++++++++++++++++++++++ src/studyClient.ts | 48 +++++-------- 10 files changed, 239 insertions(+), 37 deletions(-) create mode 100644 docs/storage-search-foundation.md create mode 100644 src/searchIndex.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b952b3..23bb0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.1.26 - 2026-07-12 + +- Started Phase 5 storage/search foundation with a reusable verse search index for browser/local preview queries. +- Routed browser verse search through the cached query layer instead of open-coded filtering. +- Added storage/search foundation documentation covering resource/user data boundaries, query goals, SQLite tables, indexes, and future full-text search. + ## 0.1.25 - 2026-07-12 - Started Phase 4 shared app core cleanup by extracting workspace/session/settings/timeline schema and persistence adapters into `src/workspace.ts`. diff --git a/ROADMAP.md b/ROADMAP.md index 01c2c96..ab69452 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -127,6 +127,8 @@ Definition of done: Goal: make hundreds of thousands or millions of entries feel normal. +Current foundation note: [Storage And Search Foundation](docs/storage-search-foundation.md). + Tasks: - Keep SQLite for desktop and private/single-user workflows. @@ -135,6 +137,7 @@ Tasks: - Add proper indexes for translation id, book/chapter/verse, Strong's number, lemma, resource id, place id, and attachment relationships. - Add full-text search through SQLite FTS5 or a compatible indexed search layer. - Avoid loading entire large resources into the browser when a chapter, page, verse, Strong's entry, or search result can be loaded on demand. +- Keep the UI pointed at shared query/search interfaces so JSON packages, Tauri SQLite, hosted SQLite, and future server search can be swapped without rewriting the reader. Definition of done: diff --git a/docs/storage-search-foundation.md b/docs/storage-search-foundation.md new file mode 100644 index 0000000..e2f1fbc --- /dev/null +++ b/docs/storage-search-foundation.md @@ -0,0 +1,91 @@ +# Storage And Search Foundation + +Libre Study should query study resources through stable app-level search and resource interfaces. The UI should not care whether a result came from local JSON packages, browser indexes, Tauri SQLite, hosted SQLite, or a future server database. + +## Current Phase 5 Step + +- Browser/local preview still loads LibreBible JSON/JSONL packages. +- Verse search now goes through a cached verse search index instead of open-coded filtering. +- The index supports normal text search, reference text, `tag:` search, and Strong's-number search. +- The index is disposable and rebuildable from the loaded translation package, which keeps it compatible with later SQLite/server-backed search. + +## Resource Data vs User Data + +Public resource data belongs to LibreBible: + +- translations +- Strong's links +- lexicons and dictionaries +- footnotes and study notes +- commentaries +- maps and place metadata +- audio Bible metadata +- pronunciation metadata + +User workspace data belongs to Libre Study: + +- sessions +- notes +- visual settings +- timeline events +- tags +- highlights +- saved searches +- future collaboration state + +## Query Interface Goals + +The app should ask stable questions: + +- search this translation for text +- find verses by Strong's number +- get notes for visible verses +- get commentary for a verse or range +- get dictionary/lexicon entries for terms, lemmas, or Strong's numbers +- get maps for places connected to this verse/page +- get audio for this chapter, verse, or lemma + +The backing implementation can change without rewriting the page UI. + +## SQLite Index Plan + +SQLite remains appropriate for desktop, local, and private alpha use. It needs explicit tables and indexes instead of ad hoc file reads. + +Core resource tables: + +- `resources`: resource id, type, title, version, license, package version. +- `verses`: translation id, source verse id, book, chapter, verse, text. +- `verse_strongs`: translation id, source verse id, position, surface, Strong's number. +- `lexicon_entries`: Strong's number, language, lemma, transliteration, pronunciation, definitions. +- `verse_notes`: resource id, translation id, source verse id, note type, caller, text. +- `commentary_entries`: resource id, book, chapter, verse start, verse end, title, text. +- `dictionary_entries`: resource id, term, slug, definitions, scripture refs. +- `places`: place id, canonical name, aliases, coordinates, bounds. +- `verse_places`: source verse id or range, place id, confidence/source. +- `audio_segments`: resource id, translation/language, book, chapter, verse range, file path/url, timestamps. +- `pronunciations`: resource id, language, term, lemma, Strong's number, file path/url, source. + +Indexes: + +- `verses(translation_id, book, chapter, verse)` +- `verses(source_verse_id)` +- `verse_strongs(strong)` +- `verse_strongs(translation_id, source_verse_id)` +- `verse_notes(translation_id, source_verse_id, note_type)` +- `commentary_entries(book, chapter, verse_start, verse_end)` +- `dictionary_entries(term)` +- `places(canonical_name)` +- `verse_places(place_id)` +- `audio_segments(book, chapter, verse_start, verse_end)` +- `pronunciations(strong_number)` +- `pronunciations(lemma)` + +Full-text indexes: + +- Bible text through SQLite FTS5. +- Notes and commentaries through SQLite FTS5. +- Dictionary/lexicon definitions through SQLite FTS5. + +## Hosted Web Path + +For the private alpha, server-side SQLite is acceptable. Before public multi-user accounts and collaboration, user/account/session data should move to a server database such as Postgres while public resource packages can remain static, indexed, or separately cached. diff --git a/package-lock.json b/package-lock.json index 1f13dc6..b2ba22a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "libre-study", - "version": "0.1.25", + "version": "0.1.26", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "libre-study", - "version": "0.1.25", + "version": "0.1.26", "dependencies": { "@tauri-apps/api": "^2.11.1", "react": "^19.2.7", diff --git a/package.json b/package.json index eb271ec..2497ff1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "libre-study", "private": true, - "version": "0.1.25", + "version": "0.1.26", "type": "module", "scripts": { "dev": "vite --host 127.0.0.1", diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index f230508..54fb0e5 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -77,7 +77,7 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "app" -version = "0.1.25" +version = "0.1.26" dependencies = [ "log", "rusqlite", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 83acdee..deb5cc0 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "app" -version = "0.1.25" +version = "0.1.26" description = "A Tauri App" authors = ["you"] license = "" diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 90cf1dc..bfcef1d 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "Libre Study", - "version": "0.1.25", + "version": "0.1.26", "identifier": "org.librestudy.desktop", "build": { "frontendDist": "../dist", diff --git a/src/searchIndex.ts b/src/searchIndex.ts new file mode 100644 index 0000000..bc469eb --- /dev/null +++ b/src/searchIndex.ts @@ -0,0 +1,116 @@ +import type { Verse } from './types' + +export type VerseSearchIndex = { + search(query: string): Verse[] +} + +const strongPattern = /^[gh]\d+$/i + +export function buildVerseSearchIndex(verses: Verse[]): VerseSearchIndex { + const indexedVerses = verses.map((verse) => ({ + verse, + haystack: normalizeText(`${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`), + })) + const versesById = new Map(verses.map((verse) => [verse.id, verse])) + const tokens = new Map>() + const strongs = new Map>() + const tags = new Map>() + + for (const verse of verses) { + addTokens(tokens, verse.id, `${verse.book} ${verse.chapter} ${verse.verse} ${verse.text}`) + for (const strong of verse.strongs) { + addToIndex(strongs, strong.strong.toUpperCase(), verse.id) + addTokens(tokens, verse.id, strong.surface) + addTokens(tokens, verse.id, strong.lemma) + addTokens(tokens, verse.id, strong.transliteration) + } + for (const tag of verse.tags) { + addToIndex(tags, normalizeTag(tag.label), verse.id) + } + } + + return { + search(query: string) { + const trimmed = query.trim() + if (!trimmed) return verses + + const lowered = trimmed.toLowerCase() + if (lowered.startsWith('tag:')) { + return verseIdsToVerses(tags.get(normalizeTag(trimmed.slice(4))) ?? new Set(), versesById) + } + + if (strongPattern.test(trimmed)) { + return verseIdsToVerses(strongs.get(trimmed.toUpperCase()) ?? new Set(), versesById) + } + + const normalizedQuery = normalizeText(trimmed) + const queryTokens = tokenize(normalizedQuery) + if (queryTokens.length) { + const candidateIds = intersectTokenMatches(tokens, queryTokens) + if (candidateIds) { + const candidates = verseIdsToVerses(candidateIds, versesById) + if (queryTokens.length === 1) return candidates + return candidates.filter((verse) => + normalizeText(`${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}`).includes(normalizedQuery), + ) + } + } + + return indexedVerses + .filter((entry) => entry.haystack.includes(normalizedQuery)) + .map((entry) => entry.verse) + }, + } +} + +function addTokens(index: Map>, verseId: number, value: string) { + for (const token of tokenize(normalizeText(value))) { + addToIndex(index, token, verseId) + } +} + +function addToIndex(index: Map>, key: string, verseId: number) { + if (!key) return + const current = index.get(key) ?? new Set() + current.add(verseId) + index.set(key, current) +} + +function intersectTokenMatches(index: Map>, queryTokens: string[]) { + let match: Set | null = null + + for (const token of queryTokens) { + const tokenMatches = index.get(token) + if (!tokenMatches) return null + if (!match) { + match = new Set(tokenMatches) + continue + } + + match = new Set([...match].filter((id) => tokenMatches.has(id))) + if (!match.size) return match + } + + return match +} + +function verseIdsToVerses(ids: Set, versesById: Map) { + const verses: Verse[] = [] + for (const id of ids) { + const verse = versesById.get(id) + if (verse) verses.push(verse) + } + return verses +} + +function tokenize(value: string) { + return value.match(/[a-z0-9]+/g) ?? [] +} + +function normalizeTag(value: string) { + return value.trim().toLowerCase() +} + +function normalizeText(value: string) { + return value.toLowerCase().replace(/[^a-z0-9\s:]/g, ' ').replace(/\s+/g, ' ').trim() +} diff --git a/src/studyClient.ts b/src/studyClient.ts index 4e701da..032486b 100644 --- a/src/studyClient.ts +++ b/src/studyClient.ts @@ -1,4 +1,5 @@ import { invoke } from '@tauri-apps/api/core' +import { buildVerseSearchIndex } from './searchIndex' import type { CommentaryEntry, DictionaryEntry, @@ -110,7 +111,6 @@ type BrowserLexiconEntry = { type BrowserStrongData = { linksByVerse: Map - verseIdsByStrong: Map> } let browserCatalogCache: Promise | null = null @@ -120,6 +120,7 @@ const browserFootnotesCache = new Map>>( const browserStudyNotesCache = new Map>>() const browserChapterFootnotesCache = new Map>>() const browserChapterStudyNotesCache = new Map>>() +const browserSearchIndexCache = new Map>>() let browserCommentaryCache: Promise> | null = null let browserDictionaryCache: Promise> | null = null const browserVerseSourceIds = new Map() @@ -359,30 +360,8 @@ export async function searchVerses(query: string, translationId = defaultTransla return invoke('search_verses', { query }) } - const trimmed = query.trim().toLowerCase() - const verses = await loadBrowserVersesForTranslation(translationId) - if (!trimmed) return verses - - if (trimmed.startsWith('tag:')) { - const label = trimmed.slice(4) - return verses.filter((verse) => - verse.tags.some((tag) => tag.label.toLowerCase() === label), - ) - } - - if (/^[gh]\d+$/.test(trimmed)) { - const strongData = await loadBrowserStrongData(translationId) - const matchingVerseIds = strongData.verseIdsByStrong.get(trimmed.toUpperCase()) ?? new Set() - return verses.filter((verse) => - matchingVerseIds.has(browserVerseKey(verse)), - ) - } - - return verses.filter((verse) => - `${verse.book} ${verse.chapter}:${verse.verse} ${verse.text}` - .toLowerCase() - .includes(trimmed), - ) + const searchIndex = await loadBrowserSearchIndex(translationId) + return searchIndex.search(query) } export async function addTagToVerse( @@ -401,6 +380,7 @@ export async function addTagToVerse( const tag = existing ?? { id: Date.now(), label, color } tagsByVerse[verseId] = existing ? current : [...current, tag] writeBrowserTags(tagsByVerse) + browserSearchIndexCache.delete(translationId) const verse = (await loadBrowserVersesForTranslation(translationId)).find((item) => item.id === verseId) if (!verse) throw new Error(`Verse ${verseId} was not found.`) @@ -441,6 +421,17 @@ async function loadBrowserCatalog(): Promise { return browserCatalogCache } +async function loadBrowserSearchIndex(translationId = defaultTranslationId) { + if (!browserSearchIndexCache.has(translationId)) { + browserSearchIndexCache.set( + translationId, + loadBrowserVersesForTranslation(translationId).then((verses) => buildVerseSearchIndex(verses)), + ) + } + + return browserSearchIndexCache.get(translationId)! +} + async function loadLibreBibleDevVerses(translationId = defaultTranslationId): Promise { const importedVerses = await fetchJsonl( `${libreBibleDevRoot}/${translationId}/verses.jsonl`, @@ -652,7 +643,6 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId) ]), ) const linksByVerse = new Map() - const verseIdsByStrong = new Map>() for (const link of importedLinks) { const entry = lexicon.get(link.strong) @@ -668,17 +658,13 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId) const current = linksByVerse.get(link.verse_id) ?? [] current.push(strongLink) linksByVerse.set(link.verse_id, current) - - const strongMatches = verseIdsByStrong.get(link.strong) ?? new Set() - strongMatches.add(link.verse_id) - verseIdsByStrong.set(link.strong, strongMatches) } for (const links of linksByVerse.values()) { links.sort((a, b) => a.position - b.position) } - return { linksByVerse, verseIdsByStrong } + return { linksByVerse } } function strongLanguage(strong: string): 'Hebrew' | 'Greek' {