diff --git a/db/migrations/018-store-embeddings-as-blobs.sql b/db/migrations/018-store-embeddings-as-blobs.sql new file mode 100644 index 00000000..fbcb4528 --- /dev/null +++ b/db/migrations/018-store-embeddings-as-blobs.sql @@ -0,0 +1,56 @@ +-- Stores document embeddings as float32 blobs instead of JSON text. +-- +-- JSON text took 3-5x the space of the binary vector (about 20-32 KB per chunk +-- at 1536 dimensions versus 6 KB), on top of the binary copy in documents_vec. +-- sqlite-vec accepts both forms, so the vector triggers pass the blob through. +-- +-- The full-text update trigger is narrowed first: it fired on every column, +-- so converting embeddings would have rewritten the whole full-text index. + +-- @migration-step scope full-text update trigger +-- Only content, metadata (path) and page_id (title, url) feed documents_fts. +DROP TRIGGER IF EXISTS documents_fts_after_update; + +CREATE TRIGGER documents_fts_after_update +AFTER UPDATE OF content, metadata, page_id ON documents +BEGIN + DELETE FROM documents_fts WHERE rowid = old.id; + INSERT INTO documents_fts(rowid, content, title, url, path) + SELECT new.id, new.content, p.title, p.url, json_extract(new.metadata, '$.path') + FROM pages p WHERE p.id = new.page_id; +END; + +-- @migration-step convert embeddings to float32 blobs +-- The vector triggers are dropped so the conversion does not rewrite +-- documents_vec, which already holds the same vectors in binary form. +-- Every stored value already went through sqlite-vec's parser when it was +-- written (the triggers below, and the backfill in 011), so vec_f32 accepts it. +DROP TRIGGER IF EXISTS documents_vec_after_insert; +DROP TRIGGER IF EXISTS documents_vec_after_update; + +UPDATE documents +SET embedding = vec_f32(embedding) +WHERE typeof(embedding) = 'text'; + +-- @migration-step recreate vector triggers +CREATE TRIGGER documents_vec_after_insert +AFTER INSERT ON documents +WHEN NEW.embedding IS NOT NULL +BEGIN + INSERT OR REPLACE INTO documents_vec (rowid, library_id, version_id, embedding) + SELECT NEW.id, v.library_id, v.id, NEW.embedding + FROM pages p + JOIN versions v ON p.version_id = v.id + WHERE p.id = NEW.page_id; +END; + +CREATE TRIGGER documents_vec_after_update +AFTER UPDATE OF embedding, page_id ON documents +BEGIN + DELETE FROM documents_vec WHERE rowid = OLD.id; + INSERT OR REPLACE INTO documents_vec (rowid, library_id, version_id, embedding) + SELECT NEW.id, v.library_id, v.id, NEW.embedding + FROM pages p + JOIN versions v ON p.version_id = v.id + WHERE p.id = NEW.page_id AND NEW.embedding IS NOT NULL; +END; diff --git a/docs/concepts/data-storage.md b/docs/concepts/data-storage.md index 9195cc7a..b9a163a4 100644 --- a/docs/concepts/data-storage.md +++ b/docs/concepts/data-storage.md @@ -174,6 +174,10 @@ Sequential SQL migrations in `db/migrations/`: 12. `011-add-vector-triggers.sql` - FTS and vector table trigger maintenance 13. `012-add-source-content-type.sql` - Source content type tracking on pages 14. `013-create-metadata-table.sql` - Key-value metadata table for embedding model tracking +15. `014-rebuild-vector-partition-keys.sql` - Partition the vector table by library and version +16. `015-add-progress-pages-indexed.sql` - Indexed page count in version progress +17. `016-add-content-url-to-pages.sql` - Retrieval location when it differs from the page URL +18. `018-store-embeddings-as-blobs.sql` - Float32 blob embeddings and a full-text update trigger limited to indexed columns **Code Reference:** All migration files in `db/migrations/` directory @@ -239,12 +243,12 @@ Handles document lifecycle operations with normalized schema access. ### Vector Storage -Embeddings stored as BLOB in documents table: +Each embedding is stored as a float32 blob in `documents.embedding`, 4 bytes per dimension: - 1536-dimensional vectors by default (configurable via `embeddings.vectorDimension`) -- Provider-agnostic binary serialization -- NULL handling for documents without embeddings -- Direct storage eliminates need for separate vector table +- Triggers on `documents` copy the blob into the `documents_vec` sqlite-vec table, which serves KNN search +- When `documents_vec` is rebuilt, vectors missing from it are backfilled from `documents.embedding` +- NULL for documents without embeddings **Code Reference:** `src/store/types.ts` line 4 (EMBEDDINGS_VECTOR_DIMENSION constant) diff --git a/openspec/specs/database-migrations/spec.md b/openspec/specs/database-migrations/spec.md index 88b51a87..f879ef7a 100644 --- a/openspec/specs/database-migrations/spec.md +++ b/openspec/specs/database-migrations/spec.md @@ -26,13 +26,19 @@ The system SHALL execute database migrations with rollback-capable SQLite journa ### Requirement: Production SQLite settings after migrations -The system SHALL configure production SQLite settings after migration execution completes, including WAL mode, bounded WAL checkpointing, busy timeout, foreign key enforcement, and `synchronous = NORMAL`. +The system SHALL configure production SQLite settings after migration execution completes, including WAL mode, bounded WAL checkpointing, a 64 MB `journal_size_limit`, busy timeout, foreign key enforcement, and `synchronous = NORMAL`. #### Scenario: Post-migration settings are applied - **WHEN** migrations complete successfully or the schema is already up to date - **THEN** the database connection MUST be configured for WAL mode, bounded autocheckpointing, busy timeout, foreign keys, and normal synchronous durability +#### Scenario: WAL file shrinks after a large write + +- **WHEN** a large write grows the WAL file beyond 64 MB +- **AND** a later checkpoint resets the WAL +- **THEN** the WAL file MUST be truncated to at most 64 MB instead of keeping its peak size + ### Requirement: Visible migration progress The system SHALL emit diagnostic progress for each pending migration. Progress MUST include the migration index and total pending migration count, migration identifier, a visible marker for each completed execution block, total elapsed time, and a completion or failure outcome. diff --git a/src/store/DocumentStore.test.ts b/src/store/DocumentStore.test.ts index 751e0467..97aacb6a 100644 --- a/src/store/DocumentStore.test.ts +++ b/src/store/DocumentStore.test.ts @@ -142,6 +142,51 @@ describe("DocumentStore - With Embeddings", () => { }); describe("Document Storage and Retrieval", () => { + it("stores each embedding as a float32 blob identical to its indexed vector", async () => { + const originalApiKey = process.env.OPENAI_API_KEY; + try { + process.env.OPENAI_API_KEY = "test-key-for-blob-storage"; + await store.shutdown(); + const cfg = loadConfig(); + cfg.app.embeddingModel = "openai:text-embedding-3-small"; + store = new DocumentStore(":memory:", cfg); + await store.initialize(); + + await store.addDocuments( + "bloblib", + "1.0.0", + 1, + createScrapeResult( + "Blob Storage", + "https://example.com/blob-storage", + "Embeddings are stored in their binary form", + ), + ); + + // @ts-expect-error Accessing private property for testing + const db = store.db; + const rows = db + .prepare(` + SELECT typeof(d.embedding) AS type, length(d.embedding) AS bytes, + (SELECT dv.embedding FROM documents_vec dv WHERE dv.rowid = d.id) + = d.embedding AS matchesIndex + FROM documents d + `) + .all() as Array<{ type: string; bytes: number; matchesIndex: number }>; + + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + expect(row).toEqual({ type: "blob", bytes: 1536 * 4, matchesIndex: 1 }); + } + } finally { + if (originalApiKey === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = originalApiKey; + } + } + }); + it("should store and retrieve documents with proper metadata", async () => { // Add two pages separately await store.addDocuments( diff --git a/src/store/DocumentStore.ts b/src/store/DocumentStore.ts index 3455b6e9..5670455b 100644 --- a/src/store/DocumentStore.ts +++ b/src/store/DocumentStore.ts @@ -149,7 +149,7 @@ export class DocumentStore { // Updated for new schema - documents table now uses page_id insertDocument: Database.Statement<[number, string, string, number]>; // Updated for new schema - embeddings stored directly in documents table - insertEmbedding: Database.Statement<[string, bigint]>; + insertEmbedding: Database.Statement<[Buffer, bigint]>; // New statement for pages table insertPage: Database.Statement< [ @@ -301,7 +301,7 @@ export class DocumentStore { private prepareStatements(): void { const statements = { getById: this.db.prepare<[bigint]>( - `SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url + `SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id WHERE d.id = ?`, @@ -310,7 +310,7 @@ export class DocumentStore { insertDocument: this.db.prepare<[number, string, string, number]>( "INSERT INTO documents (page_id, content, metadata, sort_order) VALUES (?, ?, ?, ?)", ), - insertEmbedding: this.db.prepare<[string, bigint]>( + insertEmbedding: this.db.prepare<[Buffer, bigint]>( "UPDATE documents SET embedding = ? WHERE id = ?", ), insertPage: this.db.prepare< @@ -427,7 +427,7 @@ export class DocumentStore { getChildChunks: this.db.prepare< [string, string, string, number, string, bigint, number] >(` - SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d + SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id JOIN libraries l ON v.library_id = l.id @@ -443,7 +443,7 @@ export class DocumentStore { getPrecedingSiblings: this.db.prepare< [string, string, string, bigint, string, number] >(` - SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d + SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id JOIN libraries l ON v.library_id = l.id @@ -458,7 +458,7 @@ export class DocumentStore { getSubsequentSiblings: this.db.prepare< [string, string, string, bigint, string, number] >(` - SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d + SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id JOIN libraries l ON v.library_id = l.id @@ -471,7 +471,7 @@ export class DocumentStore { LIMIT ? `), getParentChunk: this.db.prepare<[string, string, string, string, bigint]>(` - SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d + SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id JOIN libraries l ON v.library_id = l.id @@ -1260,12 +1260,12 @@ export class DocumentStore { d.id, v.library_id, v.id, - json_extract(d.embedding, '$') + d.embedding FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id WHERE d.embedding IS NOT NULL - AND vec_length(json_extract(d.embedding, '$')) = ? + AND vec_length(d.embedding) = ? AND NOT EXISTS ( SELECT 1 FROM documents_vec existing WHERE existing.rowid = d.id ) @@ -2109,7 +2109,7 @@ export class DocumentStore { // Insert into vector table only if vector search is enabled if (this.isVectorSearchEnabled && paddedEmbeddings.length > 0) { this.statements.insertEmbedding.run( - JSON.stringify(paddedEmbeddings[docIndex]), + Buffer.from(new Float32Array(paddedEmbeddings[docIndex]).buffer), BigInt(rowId), ); } @@ -2655,7 +2655,7 @@ export class DocumentStore { // Use parameterized query for variable number of IDs const placeholders = ids.map(() => "?").join(","); const stmt = this.db.prepare( - `SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d + `SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id JOIN libraries l ON v.library_id = l.id @@ -2687,7 +2687,7 @@ export class DocumentStore { try { const normalizedVersion = normalizeVersionLabel(version); const stmt = this.db.prepare( - `SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.embedding, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d + `SELECT d.id, d.page_id, d.content, json(d.metadata) as metadata, d.sort_order, d.created_at, p.url, p.title, p.source_content_type, p.content_type, p.content_url FROM documents d JOIN pages p ON d.page_id = p.id JOIN versions v ON p.version_id = v.id JOIN libraries l ON v.library_id = l.id diff --git a/src/store/applyMigrations.test.ts b/src/store/applyMigrations.test.ts index 75bf58de..20a9bb2e 100644 --- a/src/store/applyMigrations.test.ts +++ b/src/store/applyMigrations.test.ts @@ -1086,4 +1086,95 @@ describe("Database Migrations", () => { expect(bestExactScore).toBeLessThanOrEqual(bestPartialScore); } }); + + describe("embedding storage", () => { + const vector = new Array(1536).fill(0).map((_, index) => (index === 2 ? 1 : 0)); + + /** Inserts one document on a fresh library, version and page. */ + function insertDocument(content: string, embedding: string | null = null) { + db.prepare("INSERT INTO libraries (name) VALUES (?)").run("blob-lib"); + const { id: libraryId } = db + .prepare("SELECT id FROM libraries WHERE name = ?") + .get("blob-lib") as { id: number }; + db.prepare("INSERT INTO versions (library_id, name) VALUES (?, ?)").run( + libraryId, + "1.0.0", + ); + const { id: versionId } = db + .prepare("SELECT id FROM versions WHERE library_id = ?") + .get(libraryId) as { id: number }; + const pageId = db + .prepare("INSERT INTO pages (version_id, url, title) VALUES (?, ?, ?)") + .run(versionId, "https://example.com/blob", "Blob").lastInsertRowid; + const docId = db + .prepare( + "INSERT INTO documents (page_id, content, metadata, sort_order, embedding) VALUES (?, ?, ?, ?, ?)", + ) + .run( + pageId, + content, + JSON.stringify({ path: "/blob" }), + 0, + embedding, + ).lastInsertRowid; + return { libraryId, versionId, docId: Number(docId) }; + } + + it("should convert JSON embeddings to float32 blobs and keep them searchable", async () => { + await expect(applyMigrations(db)).resolves.toBeUndefined(); + // Rows written before migration 018 hold JSON text + const { libraryId, versionId, docId } = insertDocument( + "Stored as JSON", + JSON.stringify(vector), + ); + db.prepare("DELETE FROM _schema_migrations WHERE id = ?").run( + "018-store-embeddings-as-blobs.sql", + ); + + await expect(applyMigrations(db)).resolves.toBeUndefined(); + + const { embedding } = db + .prepare("SELECT embedding FROM documents WHERE id = ?") + .get(docId) as { embedding: Buffer }; + expect(Buffer.isBuffer(embedding)).toBe(true); + expect(Array.from(new Float32Array(Uint8Array.from(embedding).buffer))).toEqual( + vector, + ); + + const nearest = db + .prepare(` + SELECT rowid, distance + FROM documents_vec + WHERE library_id = ? AND version_id = ? AND embedding MATCH ? AND k = 1 + `) + .get(libraryId, versionId, JSON.stringify(vector)) as + | { rowid: number; distance: number } + | undefined; + expect(nearest?.rowid).toBe(docId); + expect(nearest?.distance).toBeCloseTo(0, 6); + }); + + it("should reindex full text when document content changes", async () => { + await expect(applyMigrations(db)).resolves.toBeUndefined(); + const { docId } = insertDocument("original wording"); + + db.prepare("UPDATE documents SET content = ? WHERE id = ?").run( + "revised wording", + docId, + ); + + const matches = (term: string) => + db + .prepare("SELECT rowid FROM documents_fts WHERE documents_fts MATCH ?") + .all(term); + expect(matches("revised")).toEqual([{ rowid: docId }]); + expect(matches("original")).toEqual([]); + }); + }); + + it("should cap the WAL file size kept after checkpoints", async () => { + await expect(applyMigrations(db)).resolves.toBeUndefined(); + + expect(db.pragma("journal_size_limit", { simple: true })).toBe(64 * 1024 * 1024); + }); }); diff --git a/src/store/applyMigrations.ts b/src/store/applyMigrations.ts index dc73e916..c60dfcf7 100644 --- a/src/store/applyMigrations.ts +++ b/src/store/applyMigrations.ts @@ -12,6 +12,7 @@ const MIGRATION_STEP_MARKER = /^\s*--\s*@migration-step\s+(.+?)\s*$/; const VECTOR_PARTITION_MIGRATION = "014-rebuild-vector-partition-keys.sql"; const DEFAULT_VECTOR_DIMENSION = 1536; const VECTOR_DIMENSION_TOKEN = "__DOCUMENTS_VEC_DIMENSION__"; +const WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024; interface MigrationStep { label: string; @@ -303,6 +304,11 @@ export async function applyMigrations( // Configure WAL autocheckpoint to prevent unbounded growth db.pragma("wal_autocheckpoint = 1000"); // Checkpoint every 1000 pages (~4MB) + // SQLite reuses the WAL file after a checkpoint but never shrinks it, so one + // large write (a bulk scrape, a migration, VACUUM) would leave it at its peak + // size until the last connection closes. Trim it back once the WAL resets. + db.pragma(`journal_size_limit = ${WAL_SIZE_LIMIT_BYTES}`); + // Set busy timeout for better handling of concurrent access db.pragma("busy_timeout = 30000"); // 30 seconds @@ -313,7 +319,7 @@ export async function applyMigrations( db.pragma("synchronous = NORMAL"); logger.debug( - "Applied production database configuration (WAL mode, autocheckpoint, foreign keys, busy timeout)", + "Applied production database configuration (WAL mode, autocheckpoint, WAL size limit, foreign keys, busy timeout)", ); } catch (_error) { logger.warn("⚠️ Could not apply all production database settings"); diff --git a/src/store/assembly/strategies/HierarchicalAssemblyStrategy.test.ts b/src/store/assembly/strategies/HierarchicalAssemblyStrategy.test.ts index c72b1a15..8c40c653 100644 --- a/src/store/assembly/strategies/HierarchicalAssemblyStrategy.test.ts +++ b/src/store/assembly/strategies/HierarchicalAssemblyStrategy.test.ts @@ -344,7 +344,6 @@ describe("HierarchicalAssemblyStrategy", () => { level: 2, }, sort_order: 1, - embedding: null, created_at: new Date().toISOString(), score: null, }, @@ -360,7 +359,6 @@ describe("HierarchicalAssemblyStrategy", () => { level: 2, }, sort_order: 3, - embedding: null, created_at: new Date().toISOString(), score: null, }, @@ -456,7 +454,6 @@ describe("HierarchicalAssemblyStrategy", () => { level: 2, }, sort_order: 0, - embedding: null, created_at: new Date().toISOString(), score: null, }, @@ -472,7 +469,6 @@ describe("HierarchicalAssemblyStrategy", () => { level: 2, }, sort_order: 0, - embedding: null, created_at: new Date().toISOString(), score: null, }, diff --git a/src/store/types.ts b/src/store/types.ts index 59714876..30d22162 100644 --- a/src/store/types.ts +++ b/src/store/types.ts @@ -48,7 +48,6 @@ export interface DbChunk { content: string; metadata: DbChunkMetadata; // Chunk-specific metadata (level, path, etc.) sort_order: number; - embedding: Buffer | null; // Binary blob for embeddings created_at: string; score: number | null; // Added during search queries }