diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f76062..424ee58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,15 @@ ## Unreleased -- Recorded the active development direction: use the local HotView/hot-reload setup for iterative app work and defer web/local builds and hosted deployment unless explicitly requested. -- Started Phase 4 shared app core cleanup by adding an explicit `StudyDataClient` contract and browser/Tauri data adapters behind the existing study client API. -- Added explicit workspace/auth client contracts so local HotView storage and hosted workspace persistence sit behind the same app-core boundary. -- Routed the app shell's hosted-login checks through the auth client instead of reading the hosted environment flag directly. -- Reframed the roadmap into seven sequential phases with a one-active-phase rule and marked hosted/deploy work as parked supporting infrastructure. -- Added a versioned workspace snapshot with migration from legacy local session/settings keys. +- No unreleased changes. + +## 0.1.32 - 2026-07-15 + +- Completed Phase 5 storage/search foundation for the active HotView workflow. +- Added a page-level data-client method so visible pages can hydrate through the shared app contract instead of assuming the reader owns whole-resource arrays. +- Changed browser/HotView Strong's loading to prefer KJV chapter shards and a compact Strong's search index, with whole-file loading kept only as a legacy package fallback. +- Added SQLite indexes and an FTS5 verse search table for the native data path. +- Updated LibreBible package consumption expectations around chapter-sharded verses, chapter-sharded Strong's links, and indexed Strong's lookups. ## 0.1.31 - 2026-07-15 diff --git a/ROADMAP.md b/ROADMAP.md index 5793fe2..9183b63 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -53,7 +53,7 @@ Work one phase at a time. A helper, spike, or supporting infrastructure task for - Phase 2: LibreBible resource consumption. Complete for the first local study workspace. - Phase 3: HotView development loop and Gitea sync discipline. Complete. - Phase 4: Shared app core. Complete. -- Phase 5: Storage and search foundation. +- Phase 5: Storage and search foundation. Complete. - Phase 6: Notes, exports, and optional sync. - Phase 7: Study resource depth and polish. @@ -135,6 +135,8 @@ Goal: make hundreds of thousands or millions of entries feel normal. Current foundation note: [Storage And Search Foundation](docs/storage-search-foundation.md). +Current status: Complete for the active HotView workflow. Browser page loads now use a shared page-query method and hydrate visible KJV Strong's links from chapter shards. Strong's-number search uses a compact package index where available. The native SQLite path now has explicit reference/attachment indexes and an FTS5 verse-search table. + Tasks: - Keep SQLite for desktop and private/single-user workflows. diff --git a/docs/storage-search-foundation.md b/docs/storage-search-foundation.md index e2f1fbc..d8b5a32 100644 --- a/docs/storage-search-foundation.md +++ b/docs/storage-search-foundation.md @@ -2,12 +2,15 @@ Libre Study should query study resources through stable app-level search and resource interfaces. The UI should not care whether a result came from local JSON packages, browser indexes, Tauri SQLite, hosted SQLite, or a future server database. -## Current Phase 5 Step +## Current Phase 5 Result - Browser/local preview still loads LibreBible JSON/JSONL packages. +- Visible page hydration now goes through a shared page-query method instead of the reader assuming every resource is already attached to the whole-library array. - Verse search now goes through a cached verse search index instead of open-coded filtering. -- The index supports normal text search, reference text, `tag:` search, and Strong's-number search. -- The index is disposable and rebuildable from the loaded translation package, which keeps it compatible with later SQLite/server-backed search. +- KJV Strong's-number search uses a compact package index where available. +- KJV page Strong's links load from `strongs-by-chapter` shards where available, with whole-file loading kept as a fallback for older packages. +- The native SQLite path has explicit reference/attachment indexes and an FTS5 verse-search table. +- Browser indexes are disposable and rebuildable from loaded package surfaces, which keeps them compatible with later SQLite/server-backed search. ## Resource Data vs User Data diff --git a/package-lock.json b/package-lock.json index a9933c9..a245067 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "libre-study", - "version": "0.1.31", + "version": "0.1.32", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "libre-study", - "version": "0.1.31", + "version": "0.1.32", "dependencies": { "@tauri-apps/api": "^2.11.1", "jszip": "^3.10.1", diff --git a/package.json b/package.json index 57cce3e..735f75e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "libre-study", "private": true, - "version": "0.1.31", + "version": "0.1.32", "type": "module", "scripts": { "dev": "vite --host 0.0.0.0", diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 54fb0e5..299add7 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -77,7 +77,7 @@ checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "app" -version = "0.1.26" +version = "0.1.32" dependencies = [ "log", "rusqlite", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index deb5cc0..84e0ffe 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "app" -version = "0.1.26" +version = "0.1.32" description = "A Tauri App" authors = ["you"] license = "" diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 986689e..9714d9b 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -226,6 +226,7 @@ fn open_db(app: &AppHandle) -> Result { let mut conn = Connection::open(path).map_err(|err| err.to_string())?; migrate(&conn)?; seed(&mut conn)?; + sync_search_indexes(&conn)?; Ok(conn) } @@ -297,6 +298,20 @@ fn migrate(conn: &Connection) -> Result<(), String> { FOREIGN KEY (verse_id) REFERENCES verses(id), FOREIGN KEY (tag_id) REFERENCES tags(id) ); + + CREATE VIRTUAL TABLE IF NOT EXISTS verses_fts USING fts5( + text, + reference, + content='verses', + content_rowid='id' + ); + + CREATE INDEX IF NOT EXISTS idx_verses_reference ON verses(book, chapter, verse); + CREATE INDEX IF NOT EXISTS idx_verse_strongs_strong ON verse_strongs(strong); + CREATE INDEX IF NOT EXISTS idx_verse_strongs_verse ON verse_strongs(verse_id); + CREATE INDEX IF NOT EXISTS idx_verse_footnotes_verse ON verse_footnotes(verse_id); + CREATE INDEX IF NOT EXISTS idx_verse_tags_tag ON verse_tags(tag_id); + CREATE INDEX IF NOT EXISTS idx_tags_label ON tags(label); ", ) .map_err(|err| err.to_string())?; @@ -778,20 +793,28 @@ fn load_verses(conn: &Connection, query: Option<&str>) -> Result, Str match query { Some(query) => { + let fts_query = search_query_for_fts(query); + if fts_query.is_empty() { + return Ok(Vec::new()); + } let pattern = format!("%{}%", query); let mut stmt = conn .prepare( " + SELECT v.id, v.book, v.chapter, v.verse, v.text + FROM verses v + JOIN verses_fts fts ON fts.rowid = v.id + WHERE verses_fts MATCH ?1 + UNION SELECT id, book, chapter, verse, text FROM verses - WHERE text LIKE ?1 - OR book || ' ' || chapter || ':' || verse LIKE ?1 + WHERE book || ' ' || chapter || ':' || verse LIKE ?2 ORDER BY id ", ) .map_err(|err| err.to_string())?; let rows = stmt - .query_map(params![pattern], row_to_verse_shell) + .query_map(params![fts_query, pattern], row_to_verse_shell) .map_err(|err| err.to_string())?; for row in rows { @@ -1110,6 +1133,30 @@ fn load_all_tags(conn: &Connection) -> Result, String> { Ok(tags) } +fn sync_search_indexes(conn: &Connection) -> Result<(), String> { + let verse_count: i64 = conn + .query_row("SELECT COUNT(*) FROM verses", [], |row| row.get(0)) + .map_err(|err| err.to_string())?; + let fts_count: i64 = conn + .query_row("SELECT COUNT(*) FROM verses_fts", [], |row| row.get(0)) + .map_err(|err| err.to_string())?; + + if verse_count != fts_count { + conn.execute("INSERT INTO verses_fts(verses_fts) VALUES('rebuild')", []) + .map_err(|err| err.to_string())?; + } + + Ok(()) +} + +fn search_query_for_fts(query: &str) -> String { + query + .split(|char: char| !char.is_ascii_alphanumeric()) + .filter(|token| !token.is_empty()) + .collect::>() + .join(" ") +} + fn looks_like_strongs(query: &str) -> bool { let mut chars = query.chars(); matches!(chars.next(), Some('G' | 'g' | 'H' | 'h')) && chars.all(|char| char.is_ascii_digit()) diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index bfcef1d..0cf183c 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "Libre Study", - "version": "0.1.26", + "version": "0.1.32", "identifier": "org.librestudy.desktop", "build": { "frontendDist": "../dist", diff --git a/src/App.tsx b/src/App.tsx index c07bad3..e11c539 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -7,6 +7,7 @@ import { getDictionaryEntriesForTerms, getLibraryState, getMapContextForVerses, + getPageVerses, getTranslationNotesForVerses, getTranslations, searchVerses, @@ -62,6 +63,17 @@ function replaceChangedVerses(verses: Verse[], hydrate: (verse: Verse) => Verse) return changed ? next : verses } +function sameHydratedVerse(left: Verse | undefined, right: Verse) { + return Boolean( + left && + left.text === right.text && + (left.strongs?.length ?? 0) === (right.strongs?.length ?? 0) && + (left.footnotes?.length ?? 0) === (right.footnotes?.length ?? 0) && + (left.studyNotes?.length ?? 0) === (right.studyNotes?.length ?? 0) && + left.tags.length === right.tags.length, + ) +} + function formatTimelineTime(value: string) { return new Date(value).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' }) } @@ -559,8 +571,8 @@ function App() { const selectedVerse = useMemo( () => - libraryVerses.find((verse) => verse.id === selectedId) ?? verses.find((verse) => verse.id === selectedId) ?? + libraryVerses.find((verse) => verse.id === selectedId) ?? verses[0], [libraryVerses, selectedId, verses], ) @@ -639,7 +651,7 @@ function App() { return anchors }, [libraryVerses, verses]) - const pageVerses = useMemo(() => { + const pageShells = useMemo(() => { if (!selectedVerse) return [] const source = libraryVerses.length ? libraryVerses : verses const pageStartIndex = source.findIndex((verse) => verse.id === pageStartId) @@ -650,6 +662,37 @@ function App() { .slice(start, start + pageSize) .filter((verse) => verse.book === book) }, [libraryVerses, pageStartId, selectedVerse, verses]) + const pageVerses = useMemo(() => { + const hydratedVerses = new Map(verses.map((verse) => [verse.id, verse])) + return pageShells.map((verse) => hydratedVerses.get(verse.id) ?? verse) + }, [pageShells, verses]) + + useEffect(() => { + if (!activeSession?.baseTranslation || !pageStartId || pageShells.length === 0) return + + let cancelled = false + getPageVerses(activeSession.baseTranslation, pageStartId, pageSize) + .then((nextPageVerses) => { + if (cancelled || nextPageVerses.length === 0) return + setVerses((current) => { + const merged = new Map(current.map((verse) => [verse.id, verse])) + let changed = false + for (const verse of nextPageVerses) { + const existing = merged.get(verse.id) + if (!sameHydratedVerse(existing, verse)) { + merged.set(verse.id, verse) + changed = true + } + } + return changed ? [...merged.values()].sort((left, right) => left.id - right.id) : current + }) + }) + .catch((reason) => setError(String(reason))) + + return () => { + cancelled = true + } + }, [activeSession?.baseTranslation, pageStartId, pageShells.length]) useEffect(() => { let cancelled = false diff --git a/src/studyClient.ts b/src/studyClient.ts index 80f4d51..c90c4ae 100644 --- a/src/studyClient.ts +++ b/src/studyClient.ts @@ -38,6 +38,8 @@ type ImportedVerse = { type ImportedStrongLink = { verse_id: string + book_id?: string + chapter?: number position: number surface: string strong: string @@ -94,6 +96,12 @@ type ImportedStrongEntry = { kjv_definition?: string } +type ImportedStrongIndexEntry = { + strong: string + verse_ids: string[] + surfaces: string[] +} + type ImportedCommentaryEntry = { id: string resource_id: string @@ -168,6 +176,9 @@ type BrowserStrongData = { let browserCatalogCache: Promise | null = null const browserLibraryCache = new Map>() const browserStrongDataCache = new Map>() +const browserChapterStrongDataCache = new Map>() +const browserStrongSearchIndexCache = new Map>>() +let browserLexiconCache: Promise> | null = null const browserFootnotesCache = new Map>>() const browserStudyNotesCache = new Map>>() const browserChapterFootnotesCache = new Map>>() @@ -179,6 +190,7 @@ let browserVersePlacesCache: Promise> | null = null let browserRoutesCache: Promise | null = null let browserAudioPlaylistCache: Promise> | null = null const browserVerseSourceIds = new Map() +const browserVerseIdsBySource = new Map() const browserAudioZipCache = new Map>() const browserAudioObjectUrlCache = new Map>() @@ -329,20 +341,48 @@ async function loadBrowserVersesForTranslation(translationId = defaultTranslatio } const verses = await browserLibraryCache.get(translationId)! - const strongData = await loadBrowserStrongData(translationId).catch((error) => { - console.warn('Unable to load browser Strong data.', error) - return null - }) const tagsByVerse = readBrowserTags() return verses.map((verse) => ({ ...verse, - strongs: strongData?.linksByVerse.get(browserVerseKey(verse)) ?? verse.strongs, footnotes: verse.footnotes ?? [], studyNotes: verse.studyNotes ?? [], tags: tagsByVerse[verse.id] ?? [], })) } +async function hydrateBrowserVerses(translationId: string, verses: Verse[]) { + const chapterKeys = [ + ...new Set( + verses + .map((verse) => `${bookIdFromVerse(verse)}.${verse.chapter}`) + .filter((key) => !key.startsWith('unknown.')), + ), + ] + const strongDataMaps = await Promise.all( + chapterKeys.map((chapterKey) => + loadBrowserChapterStrongData(translationId, chapterKey).catch((error) => { + console.warn(`Unable to load browser Strong data for ${chapterKey}.`, error) + return { linksByVerse: new Map() } + }), + ), + ) + const tagsByVerse = readBrowserTags() + + return verses.map((verse) => { + const verseKey = browserVerseKey(verse) + const strongs = + strongDataMaps.find((data) => data.linksByVerse.has(verseKey))?.linksByVerse.get(verseKey) ?? + verse.strongs + return { + ...verse, + strongs, + footnotes: verse.footnotes ?? [], + studyNotes: verse.studyNotes ?? [], + tags: tagsByVerse[verse.id] ?? [], + } + }) +} + function allBrowserTags() { const unique = new Map() for (const tags of Object.values(readBrowserTags())) { @@ -375,6 +415,23 @@ async function getBrowserLibraryState(translationId = defaultTranslationId): Pro } } +async function getBrowserPageVerses( + translationId: string, + pageStartId: number, + pageSize: number, +): Promise { + const verses = await loadBrowserVersesForTranslation(translationId) + const startIndex = verses.findIndex((verse) => verse.id === pageStartId) + if (startIndex < 0) return [] + + const book = verses[startIndex].book + const pageVerses = verses + .slice(startIndex, startIndex + pageSize) + .filter((verse) => verse.book === book) + + return hydrateBrowserVerses(translationId, pageVerses) +} + async function getBrowserTranslationNotesForVerses( translationId: string, verses: Verse[], @@ -407,6 +464,14 @@ async function getBrowserTranslationNotesForVerses( } async function searchBrowserVerses(query: string, translationId = defaultTranslationId): Promise { + if (/^[gh]\d+$/i.test(query.trim())) { + const verses = await loadBrowserVersesForTranslation(translationId) + const strongIndex = await loadBrowserStrongSearchIndex(translationId) + const sourceIds = strongIndex.get(query.trim().toUpperCase()) ?? [] + const verseIds = new Set(sourceIds.map((sourceId) => browserVerseIdsBySource.get(sourceId))) + return verses.filter((verse) => verseIds.has(verse.id)) + } + const searchIndex = await loadBrowserSearchIndex(translationId) return searchIndex.search(query) } @@ -552,6 +617,7 @@ const emptyMapContext: MapContext = { const browserStudyDataClient: StudyDataClient = { getTranslations: getBrowserTranslations, getLibraryState: getBrowserLibraryState, + getPageVerses: getBrowserPageVerses, getTranslationNotesForVerses: getBrowserTranslationNotesForVerses, searchVerses: searchBrowserVerses, addTagToVerse: addBrowserTagToVerse, @@ -565,6 +631,13 @@ const browserStudyDataClient: StudyDataClient = { const tauriStudyDataClient: StudyDataClient = { getTranslations: async () => [], getLibraryState: () => invoke('get_library_state'), + getPageVerses: async (_translationId, pageStartId, pageSize) => { + const state = await invoke('get_library_state') + const startIndex = state.verses.findIndex((verse) => verse.id === pageStartId) + if (startIndex < 0) return [] + const book = state.verses[startIndex].book + return state.verses.slice(startIndex, startIndex + pageSize).filter((verse) => verse.book === book) + }, getTranslationNotesForVerses: async () => new Map(), searchVerses: (query) => invoke('search_verses', { query }), addTagToVerse: (verseId, label, color) => invoke('add_tag_to_verse', { verseId, label, color }), @@ -589,6 +662,14 @@ export function getLibraryState(translationId = defaultTranslationId): Promise { + return getStudyDataClient().getPageVerses(translationId, pageStartId, pageSize) +} + export function getTranslationNotesForVerses( translationId: string, verses: Verse[], @@ -648,15 +729,42 @@ async function loadBrowserSearchIndex(translationId = defaultTranslationId) { return browserSearchIndexCache.get(translationId)! } +async function loadBrowserStrongSearchIndex(translationId = defaultTranslationId) { + if (!browserStrongSearchIndexCache.has(translationId)) { + browserStrongSearchIndexCache.set( + translationId, + fetchJsonl(`${libreBibleDevRoot}/${translationId}/strongs-index.jsonl`) + .then((entries) => new Map(entries.map((entry) => [entry.strong.toUpperCase(), entry.verse_ids]))) + .catch(async () => { + const strongData = await loadBrowserStrongData(translationId) + const index = new Map() + for (const [verseId, links] of strongData.linksByVerse) { + for (const link of links) { + const key = link.strong.toUpperCase() + const current = index.get(key) ?? [] + if (!current.includes(verseId)) current.push(verseId) + index.set(key, current) + } + } + return index + }), + ) + } + + return browserStrongSearchIndexCache.get(translationId)! +} + async function loadLibreBibleDevVerses(translationId = defaultTranslationId): Promise { const importedVerses = await fetchJsonl( `${libreBibleDevRoot}/${translationId}/verses.jsonl`, ) browserVerseSourceIds.clear() + browserVerseIdsBySource.clear() return importedVerses.map((verse, index) => { const id = index + 1 browserVerseSourceIds.set(id, verse.id) + browserVerseIdsBySource.set(verse.id, id) return { id, book: verse.book, @@ -982,6 +1090,42 @@ function mapImportedNotes(importedFootnotes: ImportedFootnote[]): Map { + const cacheKey = `${translationId}:${chapterKey}` + if (!browserChapterStrongDataCache.has(cacheKey)) { + browserChapterStrongDataCache.set( + cacheKey, + loadLibreBibleDevChapterStrongData(translationId, chapterKey).catch(async () => { + const fullStrongData = await loadBrowserStrongData(translationId) + const linksByVerse = new Map() + for (const [verseId, links] of fullStrongData.linksByVerse) { + if (verseId.startsWith(`${chapterKey}.`)) linksByVerse.set(verseId, links) + } + return { linksByVerse } + }), + ) + } + + return browserChapterStrongDataCache.get(cacheKey)! +} + +async function loadLibreBibleDevChapterStrongData( + translationId: string, + chapterKey: string, +): Promise { + const [importedLinks, lexicon] = await Promise.all([ + fetchJsonl( + `${libreBibleDevRoot}/${translationId}/strongs-by-chapter/${chapterKey}.strongs.jsonl`, + ), + loadBrowserLexicon(), + ]) + + return mapStrongLinks(importedLinks, lexicon) +} + async function loadBrowserStrongData(translationId = defaultTranslationId): Promise { if (!browserStrongDataCache.has(translationId)) { browserStrongDataCache.set(translationId, loadLibreBibleDevStrongData(translationId)) @@ -996,7 +1140,19 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId) fetchJsonl(`${libreBibleDevRoot}/strongs-open-scriptures/entries.jsonl`), ]) - const lexicon = new Map( + return mapStrongLinks(importedLinks, mapBrowserLexicon(importedEntries)) +} + +async function loadBrowserLexicon(): Promise> { + browserLexiconCache ??= fetchJsonl( + `${libreBibleDevRoot}/strongs-open-scriptures/entries.jsonl`, + ).then(mapBrowserLexicon) + + return browserLexiconCache +} + +function mapBrowserLexicon(importedEntries: ImportedStrongEntry[]) { + return new Map( importedEntries.map((entry) => [ entry.strongs_number, { @@ -1007,6 +1163,12 @@ async function loadLibreBibleDevStrongData(translationId = defaultTranslationId) }, ]), ) +} + +function mapStrongLinks( + importedLinks: ImportedStrongLink[], + lexicon: Map, +): BrowserStrongData { const linksByVerse = new Map() for (const link of importedLinks) { diff --git a/src/studyDataClient.ts b/src/studyDataClient.ts index bfba3fc..b8fc579 100644 --- a/src/studyDataClient.ts +++ b/src/studyDataClient.ts @@ -15,6 +15,7 @@ export type TranslationNotesByVerse = Map getLibraryState(translationId?: string): Promise + getPageVerses(translationId: string, pageStartId: number, pageSize: number): Promise getTranslationNotesForVerses(translationId: string, verses: Verse[]): Promise searchVerses(query: string, translationId?: string): Promise addTagToVerse(verseId: number, label: string, color: string, translationId?: string): Promise