diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0548b4a..8330c72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,70 @@ jobs: - name: Test default features run: cargo test + # The adapter's `memory-git` feature gates the diff family, and the test + # that the family is *withheld* without it is `#[cfg(not(feature = + # "memory-git"))]`. Both the `--all-features` and default runs above + # compile that test out, so without this step it would be checked by + # nothing — a feature-gated test whose default fate is to be built by one + # job and executed by none. + # + # This is also the configuration that keeps the promise the feature + # exists for: no `git2` / `libgit2-sys` in the graph. + # `api/Cargo.toml` spells out this exact command in a comment and asks + # that the contract crate never link a storage engine, a native library, + # an HTTP client, or an async runtime. It was left as a comment, so + # nothing checked it — and a forbidden dependency arrives transitively, + # through a feature someone enabled two crates away, which is precisely + # the way nobody notices. + # + # The FORWARD form is required. `cargo tree -i -p tinymemory-api` + # discards the `-p` scope, prints the whole-workspace inverse tree, and + # exits 0 looking clean even when this crate is the one at fault. The + # manifest says so; this runs what it says. + # `cargo build --all-targets` only *compiles* an example. `AGENTS.md` + # promises `cargo run --example basic` works, and a compiled example can + # still panic on its first line — which is the state the repository was in + # before issue #18 §E7, when the command was documented and there was no + # `examples/` directory at all. + - name: Run the bundled example + run: cargo run --example basic + + - name: Assert the contract crate stays free of heavy dependencies + run: | + forbidden="$(cargo tree -p tinymemory-api -e normal,build --prefix none \ + | grep -Ei 'rusqlite|libsqlite|git2|reqwest|regex|tokio' || true)" + if [ -n "$forbidden" ]; then + echo "tinymemory-api pulled in a dependency its manifest forbids:" >&2 + echo "$forbidden" >&2 + echo >&2 + echo "The contract is what hosts compile against. It must stay free of" >&2 + echo "storage engines, native libraries, HTTP clients and async runtimes." >&2 + exit 1 + fi + + # The minimal build has to stay genuinely usable, not merely compile: + # a host that wants the ports wired and nothing retained must be able to + # bind the null driver without pulling an engine in behind it. + - name: Build and bind the minimal configuration + run: | + cargo build -p tinymemory --no-default-features + cargo test -p tinymemory --no-default-features --test null_provider + + - name: Lint and test the adapter without its optional engine features + run: | + cargo clippy -p tinymemory-tinycortex --all-targets --no-default-features -- -D warnings + cargo test -p tinymemory-tinycortex --no-default-features + + - name: Assert the default adapter build links no native git + run: | + linked="$(cargo tree -p tinymemory-tinycortex --no-default-features \ + -e normal --prefix none | grep -cE '^(git2|libgit2-sys)' || true)" + if [ "$linked" -ne 0 ]; then + echo "the default adapter build linked $linked native-git crate(s);" >&2 + echo "the memory-git feature exists to keep them out" >&2 + exit 1 + fi + # The module crate is its own workspace root (see the `exclude` note in the # root Cargo.toml), so NONE of the steps above touch it: `--all-targets`, # `--all-features` and `--workspace` all stop at the workspace boundary and diff --git a/.gitignore b/.gitignore index 055bee4..2233b9d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,12 @@ target/ **/*.rs.bk *.pdb +# Stray build artifacts. Two 3.6 MB Linux ELF executables with debug info were +# committed at the repository root before this entry existed; both were produced +# by an ad-hoc build, referenced by nothing, and carried in every clone. +/rust_out +/tmp + # Coverage output /coverage/ *.profraw diff --git a/Cargo.lock b/Cargo.lock index d401f71..8b1112e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1808,6 +1808,7 @@ dependencies = [ "serde", "serde_json", "tinymemory-api", + "tinymemory-conformance", "tokio", ] @@ -1828,6 +1829,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "tinymemory-conformance" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "serde_json", + "tinymemory-api", + "tokio", +] + [[package]] name = "tinymemory-core" version = "0.1.0" @@ -1883,10 +1895,16 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "chrono", + "log", + "serde", + "serde_json", "tinycortex", "tinymemory", "tinymemory-api", + "tinymemory-core", "tokio", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a04e829..2915028 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = [".", "api", "core", "adapters/tinycortex", "adapters/remote"] -default-members = [".", "api", "core", "adapters/tinycortex", "adapters/remote"] +members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"] +default-members = [".", "api", "core", "adapters/tinycortex", "adapters/remote", "conformance"] # `vendor/` holds engine submodules (tinycortex, tinybus, tinyagents), each of # which is its own workspace with its own lockfile. Same exclusion # `vendor/tinycortex` uses for its own nested vendor directory. @@ -72,6 +72,13 @@ serde = { version = "1", features = ["derive"] } [dev-dependencies] # The mandatory-family tests are async. tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +# `TestHostConfig`, for the driver-selection tests. A dev-dependency: the +# facade must not carry a test double into a consumer's graph. +tinymemory-api = { path = "api", features = ["test-support"] } +# The reference driver and the behavioural suite, for the workspace-level +# integration tests. A dev-dependency only: the facade must not carry a test +# harness into a consumer's dependency graph. +tinymemory-conformance = { path = "conformance" } [features] default = [] diff --git a/README.md b/README.md index 6f911dc..941c18b 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,30 @@ src/ │ binds as, and the fail-closed external-driver gate └── mandatory/ the three mandatory capability families, composed once over the `Memory` storage trait +core/ tinymemory-core — the substance: ingestion, the summary + tree, chunk storage, entities, the graph, the diff + ledger, goals, tool-memory, and the Composio sync layer. + The largest crate here by a wide margin, and the one a + real host actually depends on. Unlike `api/` it is not + dependency-light: today it links the TinyCortex engine, + a bundled SQLite, and an HTTP stack unconditionally. adapters/ ├── tinycortex/ the TinyCortex engine seen through the contract └── remote/ native HTTP dialects for Supermemory, Mem0, and Cognee +crates/ +└── tinymemory-module/ the TinyBus loadable-module driver. Excluded from the + workspace on purpose — see the note in `Cargo.toml`. vendor/ ├── tinycortex/ the engine, pinned as a submodule +├── tinyagents/ pinned TinyAgents submodule └── tinybus/ pinned TinyBus submodule ``` +Run `git submodule update --init --recursive` after cloning. Nothing in the +workspace builds without it — `core` names `tinyagents` and `tinycortex` by +path through `vendor/`, so an uninitialized checkout fails at manifest +resolution rather than at compile time, which reads as a confusing error. + ## The contract `MemoryProvider` is an object-safe trait with **three mandatory** capability diff --git a/adapters/remote/Cargo.toml b/adapters/remote/Cargo.toml index 870bd70..d78ff7e 100644 --- a/adapters/remote/Cargo.toml +++ b/adapters/remote/Cargo.toml @@ -3,7 +3,7 @@ name = "tinymemory-remote" publish = false version = "0.1.0" edition = "2021" -rust-version = "1.85" +rust-version = "1.96" license = "MIT" description = "HTTP adapters for self-hosted Supermemory, Mem0, and Cognee" repository = "https://github.com/tinyhumansai/tinymemory" diff --git a/adapters/remote/src/failure_test.rs b/adapters/remote/src/failure_test.rs new file mode 100644 index 0000000..b6631b7 --- /dev/null +++ b/adapters/remote/src/failure_test.rs @@ -0,0 +1,249 @@ +//! What the hosted adapters do when the backend does not cooperate. +//! +//! Issue #18 §E6: "backend error, timeout, and partial-page responses on every +//! remote adapter — currently zero coverage". The existing per-adapter tests all +//! drive a backend that answers correctly, which is the half that was never in +//! doubt. +//! +//! The assertion that matters is not which error comes back — the contract's +//! error type is still `anyhow` under `MemoryError::Other` here, and §A4 is what +//! makes "unsupported" distinguishable from "failed". It is that a failure comes +//! back **at all**. +//! +//! A read that answers `Ok(None)` when the backend returned 500 is saying "this +//! memory does not exist" when the truth is "I could not ask". A caller cannot +//! tell those apart, so it writes the memory again, or reports to a user that +//! their memory is gone, or — worst — a sync job treats the empty read as +//! authoritative and prunes. Nothing surfaces until much later, which is exactly +//! the failure mode that keeps `OPENCOMPANY_MEMORY=remote` gated downstream. +//! +//! Each test drives a real adapter over a real TCP socket against a double that +//! misbehaves in one specific way, matching the harness the happy-path tests +//! already use. + +#![allow(clippy::expect_used, clippy::panic)] + +use axum::http::StatusCode; +use axum::routing::{any, get}; +use axum::Router; +use tinymemory_api::provider::MemoryProvider; +use tinymemory_api::recall::RecallOpts; +use tinymemory_api::traits::Memory; +use tinymemory_api::types::{MemoryCategory, MemoryTaint}; + +use crate::{CogneeMemory, Mem0Memory, SupermemoryMemory}; + +/// Serves `app` on an ephemeral port and returns its base URL. +async fn serve(app: Router) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + endpoint +} + +/// A backend that fails every route with `status`. +async fn failing(status: StatusCode) -> String { + serve(Router::new().fallback(any(move || async move { status }))).await +} + +/// A backend that answers every route with `200 OK` and a body that is not the +/// JSON the adapter expects. +/// +/// Distinct from an HTTP failure: the transport succeeded, so an adapter that +/// only checks the status code reaches its deserializer with rubbish. +async fn malformed() -> String { + serve(Router::new().fallback(any(|| async { "this is not the JSON you asked for" }))).await +} + +/// Every adapter, as a `Memory`, built against `endpoint`. +/// +/// Boxed rather than generic so each assertion below is written once and run +/// three times — the point is that no adapter is exempt. +fn adapters(endpoint: &str) -> Vec<(&'static str, Box)> { + vec![ + ( + "supermemory", + Box::new(SupermemoryMemory::new(endpoint, None).expect("client")) as Box, + ), + ( + "mem0", + Box::new(Mem0Memory::new(endpoint, None).expect("client")), + ), + ( + "cognee", + Box::new(CogneeMemory::self_hosted(endpoint, None).expect("client")), + ), + ] +} + +#[tokio::test] +async fn a_backend_failure_on_write_is_reported_rather_than_swallowed() { + let endpoint = failing(StatusCode::INTERNAL_SERVER_ERROR).await; + for (name, memory) in adapters(&endpoint) { + let result = memory + .store_with_taint( + "ns", + "k", + "content", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await; + assert!( + result.is_err(), + "{name}: a 500 on write must not report success — a caller that \ + believes the write landed has no reason to retry it" + ); + } +} + +#[tokio::test] +async fn a_backend_failure_on_read_is_not_reported_as_absence() { + // The one that matters most. `Ok(None)` here means "no such memory", and + // the truth is "the backend is down". + let endpoint = failing(StatusCode::INTERNAL_SERVER_ERROR).await; + for (name, memory) in adapters(&endpoint) { + let result = memory.get("ns", "k").await; + let Err(error) = result else { + panic!( + "{name}: a 500 on read must not be laundered into `Ok(None)` — \ + 'I could not ask' and 'it is not there' are different answers" + ); + }; + // Assert the failure is the *backend's*, not something incidental like a + // malformed URL. Without this the test would pass for the wrong reason + // if the adapter never reached the network at all. + let rendered = format!("{error:#}"); + assert!( + rendered.contains("500"), + "{name}: expected the backend status to survive into the error, got: {rendered}" + ); + } +} + +#[tokio::test] +async fn an_unauthorized_backend_is_not_reported_as_an_empty_store() { + // A wrong or expired credential is the most likely failure in production, + // and the most dangerous one to render as "you have no memories". + let endpoint = failing(StatusCode::UNAUTHORIZED).await; + for (name, memory) in adapters(&endpoint) { + let listed = memory.list(None, None, None).await; + assert!( + listed.is_err(), + "{name}: a 401 must not present as an empty result set" + ); + + let recalled = memory.recall("anything", 10, RecallOpts::default()).await; + assert!( + recalled.is_err(), + "{name}: a 401 on recall must not present as no matches" + ); + } +} + +#[tokio::test] +async fn a_malformed_backend_response_is_an_error_and_not_a_panic() { + // `200 OK` with a body the adapter cannot parse. An adapter that unwraps + // its way through deserialization takes the caller's process down. + let endpoint = malformed().await; + for (name, memory) in adapters(&endpoint) { + let result = memory.get("ns", "k").await; + assert!( + result.is_err(), + "{name}: an unparseable 200 body must surface as an error" + ); + } +} + +#[tokio::test] +async fn an_unreachable_backend_is_reported_rather_than_hanging() { + // Nothing is listening. This is the timeout/connection-refused leg of §E6. + // Bind a port, learn its number, drop the listener: the address is now + // reliably closed rather than merely unlikely to be in use. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let endpoint = format!("http://{}", listener.local_addr().expect("address")); + drop(listener); + + for (name, memory) in adapters(&endpoint) { + let result = memory.get("ns", "k").await; + assert!( + result.is_err(), + "{name}: an unreachable backend must surface as an error" + ); + } +} + +#[tokio::test] +async fn a_paginated_export_terminates_instead_of_looping() { + // The partial-page leg of §E6. A backend that keeps answering with a page + // and a cursor would spin an exporter forever; the contract terminates on + // `next_cursor: None`, and a driver that never emits one never finishes. + // + // Driven through the bound provider rather than the raw `Memory`: + // `export_page` is a `MemoryPortability` method, and portability is a + // mandatory supertrait of `MemoryProvider`, so it is always callable — which + // is exactly why a non-terminating one is worth pinning. + let app = Router::new().fallback(get(|| async { + axum::Json(serde_json::json!({ + "memoryEntries": [], + "results": [], + "data": [], + "pagination": {"totalPages": 1} + })) + })); + let endpoint = serve(app).await; + + let providers: Vec<(&str, Box)> = vec![ + ( + "supermemory", + Box::new(crate::supermemory_provider( + SupermemoryMemory::new(&endpoint, None).expect("client"), + )) as Box, + ), + ( + "mem0", + Box::new(crate::mem0_provider( + Mem0Memory::new(&endpoint, None).expect("client"), + )), + ), + ( + "cognee", + Box::new(crate::cognee_provider( + CogneeMemory::self_hosted(&endpoint, None).expect("client"), + )), + ), + ]; + + for (name, provider) in providers { + // Bounded so a non-terminating implementation fails rather than hanging + // the whole suite. + let finished = tokio::time::timeout(std::time::Duration::from_secs(10), async { + let mut cursor: Option = None; + for page_number in 0..100usize { + let Ok(page) = provider.export_page(cursor.as_deref(), 100).await else { + // An error is an acceptable answer here; a hang is not. + return true; + }; + match page.next_cursor { + None => return true, + Some(next) => cursor = Some(next), + } + let _ = page_number; + } + false + }) + .await; + assert_eq!( + finished, + Ok(true), + "{name}: export_page never terminated — an exporter would spin here" + ); + } +} diff --git a/adapters/remote/src/lib.rs b/adapters/remote/src/lib.rs index d649e5e..dba72ed 100644 --- a/adapters/remote/src/lib.rs +++ b/adapters/remote/src/lib.rs @@ -38,3 +38,6 @@ pub fn mem0_provider(memory: Mem0Memory) -> MemoryTraitProvider { pub fn cognee_provider(memory: CogneeMemory) -> MemoryTraitProvider { MemoryTraitProvider::new(Arc::new(memory), COGNEE_DRIVER_ID) } + +#[cfg(test)] +mod failure_test; diff --git a/adapters/tinycortex/Cargo.toml b/adapters/tinycortex/Cargo.toml index dc04faf..57f28a1 100644 --- a/adapters/tinycortex/Cargo.toml +++ b/adapters/tinycortex/Cargo.toml @@ -6,7 +6,7 @@ name = "tinymemory-tinycortex" publish = false version = "0.1.0" edition = "2021" -rust-version = "1.85" +rust-version = "1.96" license = "MIT" description = "TinyCortex engine adapter for the TinyMemory contract" repository = "https://github.com/tinyhumansai/tinymemory" @@ -22,6 +22,28 @@ tinymemory-api = { path = "../../api" } # would defeat that and give a host two TinyCortex crates with two incompatible # `Memory` traits. tinycortex = { version = "0.1", default-features = false } +# The optional capability families lifted here in issue #18 §C3 delegate to +# `tinymemory-core` on a blocking thread — that is where the summary tree, +# chunk store, entities, graph and diff ledger actually live. Depending on it +# makes this adapter heavier than the mandatory-only version it replaces; §D +# feature-gates that weight once §A3 has removed the direct engine call sites +# core still has. +tinymemory-core = { path = "../../core" } +# `spawn_blocking`: every family method runs synchronous engine work off the +# async executor rather than blocking it. +tokio = { version = "1", features = ["rt"] } +# Timestamps on ingest and diff records. +chrono = { version = "0.4", features = ["serde"] } +# The provider crosses a few value types by round-tripping them through JSON +# where the engine and the contract describe the same shape under two names — +# the duplication issue #18 §A1 exists to delete. +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# Person ids are UUIDs on the engine side. +uuid = { version = "1", features = ["v4"] } +# The engine reports a failed profile write through the `log` facade rather +# than returning it, so the host can decide what to do about it. +log = "0.4" # `Memory` is an object-safe async trait. async-trait = "0.1" @@ -43,3 +65,16 @@ unwrap_used = "warn" expect_used = "warn" panic = "warn" missing_errors_doc = "warn" + +[features] +default = [] +# Git-backed diff snapshots, forwarded to `tinymemory-core`'s own gate. Off by +# default because it is what drags `git2` / `libgit2-sys` / `libz-sys` — a +# native build — into the graph. This adapter had no such dependency before +# issue #18 §C3 lifted the diff family here, and making it unconditional would +# hand every consumer a libgit2 build they never asked for. +# +# The gate reaches `capabilities()`: with the feature off the `Diff` family is +# neither advertised nor reachable, so `audit_provider` still passes. Advertising +# a family the build cannot serve is exactly what that audit exists to catch. +memory-git = ["tinymemory-core/memory-git"] diff --git a/adapters/tinycortex/src/engine/mod.rs b/adapters/tinycortex/src/engine/mod.rs new file mode 100644 index 0000000..ebfe0eb --- /dev/null +++ b/adapters/tinycortex/src/engine/mod.rs @@ -0,0 +1,2236 @@ +//! The full TinyCortex provider: every capability family the engine can serve. +//! +//! `MemoryTraitProvider` composes the three mandatory families over the +//! `Memory` storage trait and stops there, which is honest but is also why +//! anything wanting a summary tree, entities, or a diff ledger had to reach +//! past the contract to the engine directly. This module closes that gap: the +//! optional families are implemented here, against the contract, so a host +//! filtering its surface from a negotiated capability set gets the whole engine +//! rather than a third of it. +//! +//! Lifted wholesale from `crates/tinymemory-module` (issue #18 §C3), which had +//! grown these implementations because it needed them and nowhere else had +//! them. They were never module-specific — every one delegates to +//! `tinymemory-core` on a blocking thread — so the module crate keeps only its +//! bus transport and the conversion from its own config. + +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::TinycortexMemory; +use async_trait::async_trait; +use chrono::Utc; +use tinymemory::mandatory::MemoryTraitProvider; +use tinymemory_api::capabilities::Capabilities; +use tinymemory_api::chunks::Chunk; +use tinymemory_api::error::MemoryError; +use tinymemory_api::goals::GoalsDoc; +use tinymemory_api::health::MemoryHealth; +use tinymemory_api::host::{ + CloudProviderCreds, ComposioMode, LocalAiConfig, MemoryConfig, MemoryHostConfig, + MemoryTreeConfig, SchedulerGateConfig, +}; +use tinymemory_api::provider::types::{ + EntityHit, EntityRef, ExportPage, ExportRecord, ImportOutcome, IngestItem, IngestOutcome, + MaintenanceReport, SourceItem, SourceScope, +}; +// Diff-family value types, used only by the `MemoryDiff` impl below — which is +// compiled out without the git-backed snapshot store. +#[cfg(feature = "memory-git")] +use tinymemory_api::provider::types::{ChangeKind, DiffReport, SnapshotRef, SourceChange}; +use tinymemory_api::provider::{ + AddressBookSeedOutcome, ChunkDetail, ChunkEmbedding, ChunkQuery, ConversationSegment, + CoverWindowQuery, EntityMatch, EpisodicTurn, FacetType, FastRetrieveQuery, MemoryChunks, + MemoryCore, MemoryDiff, MemoryDocuments, MemoryEntities, MemoryEpisodic, MemoryGoals, + MemoryGraph, MemoryIngest, MemoryMaintenance, MemoryPeople, MemoryPortability, MemoryProfile, + MemoryProvider, MemoryRecall, MemoryRetrieval, MemorySourceSink, MemoryToolMemory, MemoryTree, + PersonHandle, PersonInteraction, PersonRecord, PersonScore, ProfileFacet, RankedPerson, + ResolvedPerson, RetrievalHit, RetrievalResponse, SourceRetrievalQuery, UserState, +}; +use tinymemory_api::recall::OwnedRecallOpts; +use tinymemory_api::tool_memory::ToolMemoryRule; +use tinymemory_api::tree::{IngestRequest, QueryResult, TreeStatus}; +use tinymemory_api::types::{ + GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, + NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext, NamespaceSummary, + StoredMemoryDocument, +}; +use tinymemory_core::store::{MemoryClient, MemoryClientRef}; + +/// The concrete, credential-free host configuration available inside a module. +#[derive(Debug, Clone)] +pub struct EngineRuntimeConfig { + /// Root of the memory workspace on disk. + pub workspace_dir: PathBuf, + /// The `config.toml` inside [`Self::workspace_dir`]. + pub config_path: PathBuf, + /// Memory engine settings. + pub memory: MemoryConfig, + /// Summary-tree settings. + pub memory_tree: MemoryTreeConfig, + /// Whether background work may run, and under what budget. + pub scheduler_gate: SchedulerGateConfig, + /// Local inference settings. + pub local_ai: LocalAiConfig, + /// Embeddings provider id, when one is configured. + pub embeddings_provider: Option, + /// Memory driver id, when the host names one. + pub memory_provider: Option, + /// Default chat model id, when one is configured. + pub default_model: Option, + /// Default sampling temperature. + pub default_temperature: f64, + /// Preferred output language, when the host sets one. + pub output_language: Option, + /// Opaque source configuration, passed through verbatim. + pub memory_sources: serde_json::Value, +} + +#[async_trait] +impl MemoryHostConfig for EngineRuntimeConfig { + fn workspace_dir(&self) -> &PathBuf { + &self.workspace_dir + } + fn config_path(&self) -> &PathBuf { + &self.config_path + } + fn memory_tree_content_root(&self) -> PathBuf { + self.memory_tree + .content_dir + .clone() + .unwrap_or_else(|| self.workspace_dir.join("memory_tree/content")) + } + fn memory(&self) -> &MemoryConfig { + &self.memory + } + fn memory_tree(&self) -> &MemoryTreeConfig { + &self.memory_tree + } + fn scheduler_gate(&self) -> &SchedulerGateConfig { + &self.scheduler_gate + } + fn local_ai(&self) -> &LocalAiConfig { + &self.local_ai + } + fn cloud_providers(&self) -> &Vec { + static NONE: Vec = Vec::new(); + &NONE + } + fn embeddings_provider(&self) -> Option<&str> { + self.embeddings_provider.as_deref() + } + fn memory_provider(&self) -> Option<&str> { + self.memory_provider.as_deref() + } + fn workload_local_model(&self, workload: &str) -> Option { + let route = match workload { + "memory" => self.memory_provider.as_deref(), + "embeddings" => self.embeddings_provider.as_deref(), + _ => None, + }?; + route + .strip_prefix("ollama:") + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn to_arc(&self) -> Arc { + Arc::new(self.clone()) + } + fn api_url(&self) -> Option<&str> { + None + } + fn effective_backend_api_url(&self) -> String { + String::new() + } + fn session_token(&self) -> Result, String> { + Ok(None) + } + fn default_model(&self) -> Option<&str> { + self.default_model.as_deref() + } + fn default_temperature(&self) -> f64 { + self.default_temperature + } + fn output_language(&self) -> Option<&str> { + self.output_language.as_deref() + } + fn memory_sync_interval_secs(&self) -> Option { + Some(0) + } + fn onboarding_completed(&self) -> bool { + true + } + fn secrets_encrypt(&self) -> bool { + false + } + fn composio(&self) -> ComposioMode { + ComposioMode::default() + } + fn memory_sources_json(&self) -> anyhow::Result { + Ok(self.memory_sources.clone()) + } + fn set_memory_sources_json(&mut self, value: serde_json::Value) -> anyhow::Result<()> { + self.memory_sources = value; + Ok(()) + } + fn composio_source_caps_migration_version(&self) -> u32 { + 0 + } + fn set_composio_source_caps_migration_version(&mut self, _version: u32) {} + fn apply_env_overrides(&mut self) {} + async fn save(&self) -> anyhow::Result<()> { + Ok(()) + } +} + +/// The module-owned implementation of every TinyMemory capability family. +pub struct TinycortexProvider { + driver_id: String, + mandatory: MemoryTraitProvider, + client: MemoryClientRef, + config: EngineRuntimeConfig, +} + +impl TinycortexProvider { + /// Binds the engine as a provider. + /// + /// `driver_id` is the id the host admitted this driver under, not something + /// the adapter chooses — see the class note in `tinymemory::registry`. + pub fn new(driver_id: String, config: EngineRuntimeConfig, client: Arc) -> Self { + let memory = client.memory_handle(); + let mandatory = + MemoryTraitProvider::new(Arc::new(TinycortexMemory::new(memory)), driver_id.clone()); + Self { + driver_id, + mandatory, + client, + config, + } + } + + fn other(context: &'static str, error: impl std::fmt::Display) -> MemoryError { + MemoryError::Other(anyhow::anyhow!("{context}: {error}")) + } + + fn cross( + value: &A, + context: &'static str, + ) -> Result { + let value = serde_json::to_value(value).map_err(|error| Self::other(context, error))?; + serde_json::from_value(value).map_err(|error| Self::other(context, error)) + } +} + +fn validate_ingest_item(item: &IngestItem) -> Result<(), MemoryError> { + if item.taint != MemoryTaint::default() { + return Err(MemoryError::Invalid( + "ingest cannot preserve a non-default taint in the chunk tier".to_string(), + )); + } + if item.content.trim().is_empty() { + return Err(MemoryError::Invalid( + "ingest content must not be empty".to_string(), + )); + } + if let Some(mime) = item.mime.as_deref() { + let mime = mime.trim().to_ascii_lowercase(); + let base = mime.split(';').next().unwrap_or("").trim(); + if !(base.starts_with("text/") + || base.ends_with("+json") + || base.ends_with("+xml") + || matches!( + base, + "application/json" | "application/xml" | "application/x-ndjson" + )) + { + return Err(MemoryError::Invalid(format!( + "unsupported MIME '{mime}': ingest accepts decoded text only" + ))); + } + } + Ok(()) +} + +async fn blocking( + config: EngineRuntimeConfig, + context: &'static str, + run: F, +) -> Result +where + T: Send + 'static, + F: FnOnce(&EngineRuntimeConfig) -> anyhow::Result + Send + 'static, +{ + tokio::task::spawn_blocking(move || run(&config)) + .await + .map_err(|error| TinycortexProvider::other(context, error))? + .map_err(|error| TinycortexProvider::other(context, error)) +} + +/// The capability families this build can actually serve. +/// +/// A free function rather than only a method because it is the rule that has to +/// stay true, and it is testable on its own — constructing a provider requires +/// a `MemoryClient`, which requires the host's process-global seams to be +/// installed, and a test that installs those is order-dependent. +/// +/// The `Diff` family is compiled out without `memory-git`, so a build without +/// it must not advertise `Diff`: `audit_provider` compares this set against the +/// reachable `as_*` accessors, and a mismatch is the failure that audit exists +/// to catch. The `#[cfg]` here and the one on [`MemoryProvider::as_diff`] are +/// the same condition on purpose. +#[must_use] +pub fn advertised_capabilities() -> Capabilities { + #[cfg(feature = "memory-git")] + { + Capabilities::all() + } + #[cfg(not(feature = "memory-git"))] + { + Capabilities::all().without(tinymemory_api::capabilities::Capability::Diff) + } +} + +#[async_trait] +impl MemoryCore for TinycortexProvider { + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + self.mandatory + .store(namespace, key, content, category, session_id, taint) + .await + } + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { + self.mandatory.get(namespace, key).await + } + async fn forget(&self, namespace: &str, key: &str) -> Result { + self.mandatory.forget(namespace, key).await + } + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError> { + self.mandatory.list(namespace, category, session_id).await + } + async fn namespaces(&self) -> Result, MemoryError> { + self.mandatory.namespaces().await + } +} + +#[async_trait] +impl MemoryRecall for TinycortexProvider { + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + self.mandatory.recall(query, limit, opts, scope).await + } +} + +#[async_trait] +impl MemoryPortability for TinycortexProvider { + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.mandatory.export_page(cursor, limit).await + } + async fn import_records( + &self, + records: Vec, + ) -> Result { + self.mandatory.import_records(records).await + } +} + +#[async_trait] +impl MemoryDocuments for TinycortexProvider { + async fn put_document(&self, input: NamespaceDocumentInput) -> Result { + let input = Self::cross(&input, "convert document input")?; + self.client + .put_doc(input) + .await + .map_err(|error| Self::other("put_document", error)) + } + async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, MemoryError> { + let document = self + .client + .get_document(namespace, key) + .await + .map_err(|error| Self::other("get_document", error))?; + document + .map(|document| Self::cross(&document, "convert stored document")) + .transpose() + } + + async fn list_documents( + &self, + namespace: Option<&str>, + ) -> Result { + self.client + .list_documents(namespace) + .await + .map_err(|error| Self::other("list_documents", error)) + } + + async fn list_namespaces(&self) -> Result, MemoryError> { + self.client + .list_namespaces() + .await + .map_err(|error| Self::other("list_namespaces", error)) + } + + async fn delete_document( + &self, + namespace: &str, + document_id: &str, + ) -> Result { + self.client + .delete_document(namespace, document_id) + .await + .map_err(|error| Self::other("delete_document", error)) + } + + async fn clear_namespace(&self, namespace: &str) -> Result<(), MemoryError> { + self.client + .clear_namespace(namespace) + .await + .map_err(|error| Self::other("clear_namespace", error)) + } + async fn query_documents( + &self, + namespace: &str, + query: &str, + limit: usize, + ) -> Result { + let limit = u32::try_from(limit).unwrap_or(u32::MAX); + let context = self + .client + .query_namespace_context_data(namespace, query, limit) + .await + .map_err(|error| Self::other("query_documents", error))?; + Self::cross(&context, "convert document query result") + } + + async fn recall_documents( + &self, + namespace: &str, + limit: usize, + ) -> Result { + let limit = u32::try_from(limit).unwrap_or(u32::MAX); + let context = self + .client + .recall_namespace_context_data(namespace, limit) + .await + .map_err(|error| Self::other("recall_documents", error))?; + Self::cross(&context, "convert document recall result") + } +} + +#[async_trait] +impl MemoryIngest for TinycortexProvider { + async fn ingest_document(&self, item: IngestItem) -> Result { + validate_ingest_item(&item)?; + let document = tinycortex::memory::ingest::canonicalize::document::DocumentInput { + provider: item.source.as_str().to_string(), + title: String::new(), + body: item.content, + modified_at: item.timestamp.unwrap_or_else(Utc::now), + source_ref: item.source_ref.map(|source_ref| source_ref.value), + }; + let result = tinymemory_core::ingest_pipeline::ingest_document_with_scope( + &self.config, + &item.source_id, + &item.owner, + item.tags, + document, + item.path_scope, + ) + .await + .map_err(|error| Self::other("ingest document", error))?; + Ok(IngestOutcome { + written: u32::try_from(result.chunks_written).unwrap_or(u32::MAX), + skipped: if result.already_ingested { + 1 + } else { + u32::try_from(result.chunks_dropped).unwrap_or(u32::MAX) + }, + ids: result.chunk_ids, + }) + } + + async fn ingest_chat(&self, messages: Vec) -> Result { + let Some(first) = messages.first() else { + return Ok(IngestOutcome::default()); + }; + let source_id = first.source_id.clone(); + let owner = first.owner.clone(); + let tags = first.tags.clone(); + let platform = first.source.as_str().to_string(); + for item in &messages { + validate_ingest_item(item)?; + if item.source_id != source_id { + return Err(MemoryError::Invalid( + "ingest_chat batches must contain one conversation".to_string(), + )); + } + } + let batch = tinycortex::memory::ingest::canonicalize::chat::ChatBatch { + platform, + channel_label: source_id.clone(), + messages: messages + .into_iter() + .map( + |item| tinycortex::memory::ingest::canonicalize::chat::ChatMessage { + author: item.owner, + timestamp: item.timestamp.unwrap_or_else(Utc::now), + text: item.content, + source_ref: item.source_ref.map(|source_ref| source_ref.value), + }, + ) + .collect(), + }; + let result = tinymemory_core::ingest_pipeline::ingest_chat( + &self.config, + &source_id, + &owner, + tags, + batch, + ) + .await + .map_err(|error| Self::other("ingest chat", error))?; + Ok(IngestOutcome { + written: u32::try_from(result.chunks_written).unwrap_or(u32::MAX), + skipped: if result.already_ingested { + 1 + } else { + u32::try_from(result.chunks_dropped).unwrap_or(u32::MAX) + }, + ids: result.chunk_ids, + }) + } +} + +#[async_trait] +impl MemoryGraph for TinycortexProvider { + async fn kv_get( + &self, + namespace: Option<&str>, + key: &str, + ) -> Result, MemoryError> { + let record = self + .client + .kv_records(namespace) + .await + .map_err(|error| Self::other("kv_get", error))? + .into_iter() + .find(|record| record.key == key); + record + .map(|record| Self::cross(&record, "convert key/value record")) + .transpose() + } + async fn kv_put( + &self, + namespace: Option<&str>, + key: &str, + value: serde_json::Value, + ) -> Result<(), MemoryError> { + self.client + .kv_set(namespace, key, &value) + .await + .map_err(|error| Self::other("kv_put", error)) + } + + async fn kv_delete(&self, namespace: Option<&str>, key: &str) -> Result { + self.client + .kv_delete(namespace, key) + .await + .map_err(|error| Self::other("kv_delete", error)) + } + async fn kv_list( + &self, + namespace: Option<&str>, + prefix: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + let mut records = self + .client + .kv_records(namespace) + .await + .map_err(|error| Self::other("kv_list", error))?; + if let Some(prefix) = prefix { + records.retain(|record| record.key.starts_with(prefix)); + } + records.truncate(limit); + Self::cross(&records, "convert key/value records") + } + async fn relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + let mut records = self + .client + .graph_relations(namespace, subject, predicate) + .await + .map_err(|error| Self::other("relations", error))?; + records.truncate(limit); + Self::cross(&records, "convert graph relations") + } + async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError> { + self.client + .graph_upsert( + relation.namespace.as_deref(), + &relation.subject, + &relation.predicate, + &relation.object, + &relation.attrs, + ) + .await + .map_err(|error| Self::other("put_relation", error)) + } +} + +#[async_trait] +impl MemoryGoals for TinycortexProvider { + async fn goals(&self) -> Result { + let workspace = self.config.workspace_dir.clone(); + let document = + tokio::task::spawn_blocking(move || tinycortex::memory::goals::store::load(&workspace)) + .await + .map_err(|error| Self::other("join goals read", error))? + .map_err(|error| Self::other("read goals", error))?; + Self::cross(&document, "convert goals") + } + + async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError> { + let workspace = self.config.workspace_dir.clone(); + let mut goals = Self::cross(&goals, "convert goals")?; + tokio::task::spawn_blocking(move || { + tinycortex::memory::goals::store::save(&workspace, &mut goals) + }) + .await + .map_err(|error| Self::other("join goals write", error))? + .map_err(|error| Self::other("write goals", error)) + } +} + +#[async_trait] +impl MemoryToolMemory for TinycortexProvider { + async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError> { + let rules = tinymemory_core::tool_memory::tool_memory_store(self.client.memory_handle()) + .list_rules(tool_name) + .await + .map_err(|error| Self::other("list tool rules", error))?; + Self::cross(&rules, "convert tool rules") + } + + async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError> { + let rule = Self::cross(&rule, "convert tool rule")?; + tinymemory_core::tool_memory::tool_memory_store(self.client.memory_handle()) + .put_rule(rule) + .await + .map(|_| ()) + .map_err(|error| Self::other("put tool rule", error)) + } + + async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result { + tinymemory_core::tool_memory::tool_memory_store(self.client.memory_handle()) + .delete_rule(tool_name, rule_id) + .await + .map_err(|error| Self::other("delete tool rule", error)) + } +} + +#[async_trait] +impl MemoryTree for TinycortexProvider { + async fn append(&self, request: IngestRequest) -> Result<(), MemoryError> { + tinycortex::memory::tree::runtime::store::validate_namespace(&request.namespace) + .map_err(MemoryError::Invalid)?; + if request.content.trim().is_empty() { + return Err(MemoryError::Invalid( + "content must not be empty".to_string(), + )); + } + let namespace = request.namespace.trim().to_string(); + let content = request.content; + let timestamp = request.timestamp.unwrap_or_else(Utc::now); + let metadata = request.metadata; + blocking(self.config.clone(), "append tree content", move |config| { + tinymemory_core::tree::tree_runtime::store::buffer_write( + config, + &namespace, + &content, + ×tamp, + metadata.as_ref(), + ) + .map(|_| ()) + }) + .await + } + + async fn query_source( + &self, + namespace: &str, + source_id: &str, + limit: usize, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + let query = tinymemory_core::store::chunks::ListChunksQuery { + source_id: Some(source_id.to_string()), + source_scope: scope.map(|scope| scope.allow.iter().cloned().collect::>()), + limit: Some(limit), + exclude_dropped: true, + ..Default::default() + }; + let chunks = blocking(self.config.clone(), "query source", move |config| { + tinymemory_core::store::chunks::list_chunks(config, &query) + }) + .await?; + Self::cross(&chunks, "convert source chunks") + } + + async fn drill_down(&self, namespace: &str, node_id: &str) -> Result { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + tinycortex::memory::tree::runtime::store::validate_node_id(node_id) + .map_err(MemoryError::Invalid)?; + let namespace = namespace.trim().to_string(); + let node_id = node_id.to_string(); + let lookup_namespace = namespace.clone(); + let lookup_node = node_id.clone(); + let result = blocking(self.config.clone(), "drill down", move |config| { + let Some(node) = tinymemory_core::tree::tree_runtime::store::read_node( + config, + &lookup_namespace, + &lookup_node, + )? + else { + return Ok(None); + }; + let children = tinymemory_core::tree::tree_runtime::store::read_children( + config, + &lookup_namespace, + &lookup_node, + )?; + Ok(Some((node, children))) + }) + .await? + .ok_or_else(|| { + MemoryError::NotFound(format!("tree node '{node_id}' not found in '{namespace}'")) + })?; + Self::cross(&result, "convert tree drill-down") + .map(|(node, children)| QueryResult { node, children }) + } + + async fn seal(&self, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + let namespace = namespace.trim().to_string(); + let read_namespace = namespace.clone(); + let buffered = blocking(self.config.clone(), "read tree buffer", move |config| { + tinymemory_core::tree::tree_runtime::store::buffer_read(config, &read_namespace) + }) + .await?; + if !buffered.is_empty() { + let (model, _) = tinymemory_core::chat_host::create_chat_model_with_model_id( + "summarization", + &self.config, + self.config.default_temperature, + ) + .map_err(|error| Self::other("create summarizer", error))?; + tinymemory_core::tree::tree_runtime::engine::run_summarization( + &self.config, + model.as_ref(), + &namespace, + Utc::now(), + ) + .await + .map_err(|error| Self::other("seal tree", error))?; + } + let status = blocking(self.config.clone(), "read tree status", move |config| { + tinymemory_core::tree::tree_runtime::store::get_tree_status(config, &namespace) + }) + .await?; + Self::cross(&status, "convert tree status") + } + + async fn cascade(&self, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + let namespace = namespace.trim().to_string(); + let read_namespace = namespace.clone(); + let status = blocking(self.config.clone(), "read tree status", move |config| { + tinymemory_core::tree::tree_runtime::store::get_tree_status(config, &read_namespace) + }) + .await?; + if status.total_nodes == 0 { + return Self::cross(&status, "convert tree status"); + } + let (model, _) = tinymemory_core::chat_host::create_chat_model_with_model_id( + "summarization", + &self.config, + self.config.default_temperature, + ) + .map_err(|error| Self::other("create summarizer", error))?; + let status = tinymemory_core::tree::tree_runtime::engine::rebuild_tree( + &self.config, + model.as_ref(), + &namespace, + ) + .await + .map_err(|error| Self::other("cascade tree", error))?; + Self::cross(&status, "convert tree status") + } +} + +#[async_trait] +impl MemoryEntities for TinycortexProvider { + async fn entities( + &self, + namespace: &str, + query: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + let namespace = namespace.to_string(); + let query_namespace = namespace.clone(); + let query = query.map(str::to_string); + let rows = blocking( + self.config.clone(), + "list namespace entities", + move |config| { + tinymemory_core::store::entities::namespace_entities( + config, + &query_namespace, + query.as_deref(), + limit, + ) + }, + ) + .await? + .into_iter() + .map(|hit| (hit.id, hit.kind, hit.name, hit.mentions)) + .collect::>(); + + let config = self.config.clone(); + blocking(config, "attach entity hotness", move |config| { + Ok(rows + .into_iter() + .map(|(id, kind, name, mentions)| { + let hotness_key = format!("{namespace}:{id}"); + let hotness = tinymemory_core::store::trees::hotness::get(config, &hotness_key) + .ok() + .flatten() + .map_or(0.0, |counters| { + f64::from( + tinymemory_core::tree_policy::TreePolicy::topic().topic_hotness( + &id, + &counters.stats(), + Utc::now().timestamp_millis(), + ), + ) + }); + EntityHit { + entity: EntityRef { id, kind, name }, + hotness, + mentions, + } + }) + .collect()) + }) + .await + } + + async fn entity_edges( + &self, + namespace: &str, + entity_id: &str, + limit: usize, + ) -> Result, MemoryError> { + let subject = entity_id.to_string(); + let lookup = subject.clone(); + let namespace = namespace.to_string(); + let query_namespace = namespace.clone(); + let neighbours = blocking(self.config.clone(), "read entity edges", move |config| { + tinymemory_core::store::entities::namespace_entity_edges( + config, + &query_namespace, + &lookup, + limit, + ) + }) + .await?; + Ok(neighbours + .into_iter() + .map(|(object, weight)| GraphRelationRecord { + namespace: Some(namespace.clone()), + subject: subject.clone(), + predicate: "co_occurs_with".to_string(), + object, + attrs: serde_json::Value::Null, + updated_at: 0.0, + evidence_count: weight, + order_index: None, + document_ids: Vec::new(), + chunk_ids: Vec::new(), + }) + .collect()) + } + + async fn touch_entities( + &self, + namespace: &str, + entity_ids: &[String], + ) -> Result<(), MemoryError> { + let entity_ids = entity_ids.to_vec(); + let namespace = namespace.to_string(); + blocking(self.config.clone(), "touch entities", move |config| { + let now = Utc::now().timestamp_millis(); + for entity_id in entity_ids { + let entity_id = format!("{namespace}:{entity_id}"); + let mut counters = + tinymemory_core::store::trees::hotness::get_or_fresh(config, &entity_id)?; + counters.mention_count_30d = counters.mention_count_30d.saturating_add(1); + counters.last_seen_ms = Some(now); + counters.last_updated_ms = now; + tinymemory_core::store::trees::hotness::upsert(config, &counters)?; + } + Ok(()) + }) + .await + } +} + +#[cfg(feature = "memory-git")] +#[async_trait] +impl MemoryDiff for TinycortexProvider { + async fn capture_snapshot(&self, source_id: &str) -> Result { + let source = tinymemory_core::sources::registry::decode_memory_sources(&self.config) + .into_iter() + .find(|source| source.id == source_id) + .ok_or_else(|| MemoryError::NotFound(source_id.to_string()))?; + let snapshot = tinymemory_core::diff::ops::take_snapshot( + &source, + &self.config, + tinymemory_core::diff::SnapshotTrigger::Manual, + ) + .await + .map_err(|error| Self::other("capture snapshot", error))?; + Ok(SnapshotRef { + id: snapshot.id, + source_id: snapshot.source_id, + label: snapshot.label, + item_count: snapshot.item_count, + taken_at_ms: snapshot.taken_at_ms, + }) + } + + async fn snapshots( + &self, + source_id: &str, + limit: usize, + ) -> Result, MemoryError> { + let snapshots = tinymemory_core::diff::ops::list_snapshots( + &self.config, + Some(source_id), + u32::try_from(limit).unwrap_or(u32::MAX), + ) + .await + .map_err(|error| Self::other("list snapshots", error))?; + Ok(snapshots + .into_iter() + .map(|snapshot| SnapshotRef { + id: snapshot.id, + source_id: snapshot.source_id, + label: snapshot.label, + item_count: snapshot.item_count, + taken_at_ms: snapshot.taken_at_ms, + }) + .collect()) + } + + async fn diff( + &self, + source_id: &str, + from: Option<&str>, + to: &str, + ) -> Result { + let result = tinymemory_core::diff::ops::compute_diff(&self.config, from, to, false) + .await + .map_err(|error| Self::other("compute diff", error))?; + if result.source_id != source_id { + return Err(MemoryError::Invalid(format!( + "snapshot '{to}' belongs to a different source" + ))); + } + let changes = result + .changes + .into_iter() + .map(|change| SourceChange { + item_id: change.item_id, + title: change.title, + kind: match change.kind { + tinymemory_core::diff::ChangeKind::Added => ChangeKind::Added, + tinymemory_core::diff::ChangeKind::Removed => ChangeKind::Removed, + tinymemory_core::diff::ChangeKind::Modified => ChangeKind::Modified, + }, + old_content_hash: change.old_content_hash, + new_content_hash: change.new_content_hash, + }) + .collect(); + Ok(DiffReport { + source_id: result.source_id, + from_snapshot_id: result.from_snapshot_id, + to_snapshot_id: result.to_snapshot_id, + added: result.summary.added, + removed: result.summary.removed, + modified: result.summary.modified, + unchanged: result.summary.unchanged, + changes, + }) + } +} + +#[async_trait] +impl MemorySourceSink for TinycortexProvider { + async fn accept_source_items( + &self, + source_id: &str, + source_kind: &str, + items: Vec, + taint: MemoryTaint, + ) -> Result { + let namespace = format!("source:{source_id}"); + let mut outcome = IngestOutcome::default(); + for item in items { + if item.item_id.trim().is_empty() { + return Err(MemoryError::Invalid( + "source item_id must not be empty".to_string(), + )); + } + let title = if item.title.trim().is_empty() { + item.item_id.clone() + } else { + item.title.clone() + }; + let input = NamespaceDocumentInput { + namespace: namespace.clone(), + key: item.item_id, + title, + content: item.content, + source_type: source_kind.to_string(), + priority: "medium".to_string(), + tags: item.tags, + metadata: serde_json::json!({ + "sourceId": source_id, + "sourceKind": source_kind, + "url": item.url, + "mime": item.mime, + "updatedAtMs": item.updated_at_ms, + }), + category: "core".to_string(), + session_id: None, + document_id: None, + taint, + }; + let input = Self::cross(&input, "convert source document")?; + match self.client.put_doc(input).await { + Ok(id) => { + outcome.written = outcome.written.saturating_add(1); + outcome.ids.push(id); + } + Err(_) => { + outcome.skipped = outcome.skipped.saturating_add(1); + } + } + } + Ok(outcome) + } + + async fn forget_source(&self, source_id: &str) -> Result { + let namespace = format!("source:{source_id}"); + let listed = self + .client + .list_documents(Some(&namespace)) + .await + .map_err(|error| Self::other("list source documents", error))?; + let documents = listed + .get("documents") + .and_then(serde_json::Value::as_array) + .map_or(0, Vec::len); + if documents > 0 { + self.client + .clear_namespace(&namespace) + .await + .map_err(|error| Self::other("clear source documents", error))?; + } + let source_id = source_id.to_string(); + let chunks = blocking(self.config.clone(), "clear source chunks", move |config| { + use tinymemory_core::store::chunks::{ + delete_chunks_by_source, delete_orphaned_source_tree, SourceKind, + }; + let removed = delete_chunks_by_source(config, SourceKind::Document, &source_id)?; + delete_orphaned_source_tree(config, SourceKind::Document, &source_id)?; + Ok(removed) + }) + .await?; + Ok(u64::try_from(documents.saturating_add(chunks)).unwrap_or(u64::MAX)) + } +} + +#[async_trait] +impl MemoryMaintenance for TinycortexProvider { + async fn reembed(&self) -> Result { + let (examined, changed) = + blocking(self.config.clone(), "enqueue re-embedding", move |config| { + let total = tinymemory_core::queue::count_total(config).unwrap_or(0); + let before = tinymemory_core::queue::count_by_status( + config, + tinymemory_core::queue::JobStatus::Ready, + ) + .unwrap_or(0); + tinymemory_core::queue::ensure_reembed_backfill(config); + let after = tinymemory_core::queue::count_by_status( + config, + tinymemory_core::queue::JobStatus::Ready, + ) + .unwrap_or(0); + Ok((total, after.saturating_sub(before))) + }) + .await?; + Ok(MaintenanceReport { + operation: "reembed".to_string(), + examined, + changed, + findings: vec![format!("enqueued {changed} re-embedding job(s)")], + }) + } + + async fn compact(&self) -> Result { + let (examined, changed) = + blocking(self.config.clone(), "compact memory queue", move |config| { + Ok(( + tinymemory_core::queue::count_total(config).unwrap_or(0), + u64::try_from(tinymemory_core::queue::recover_stale_locks(config).unwrap_or(0)) + .unwrap_or(u64::MAX), + )) + }) + .await?; + Ok(MaintenanceReport { + operation: "compact".to_string(), + examined, + changed, + findings: vec![format!("released {changed} stale queue lock(s)")], + }) + } + + async fn consolidate(&self) -> Result { + let (examined, enqueued) = blocking( + self.config.clone(), + "enqueue consolidation", + move |config| { + Ok(( + tinymemory_core::queue::count_total(config).unwrap_or(0), + tinymemory_core::queue::scheduler::enqueue_flush_stale_job(config) + .map_err(anyhow::Error::msg)?, + )) + }, + ) + .await?; + Ok(MaintenanceReport { + operation: "consolidate".to_string(), + examined, + changed: u64::from(enqueued), + findings: vec![if enqueued { + "enqueued a stale-buffer flush".to_string() + } else { + "a stale-buffer flush is already queued".to_string() + }], + }) + } + + async fn doctor(&self) -> Result { + let report = tinymemory_core::tree::health::async_run_doctor(&self.config).await; + Ok(MaintenanceReport { + operation: "doctor".to_string(), + examined: report.counters.total_chunks, + changed: 0, + findings: report + .stages + .into_iter() + .filter(|stage| !stage.ok) + .map(|stage| format!("{}: {}", stage.stage, stage.note)) + .collect(), + }) + } +} + +#[async_trait] +impl MemoryProvider for TinycortexProvider { + fn driver_id(&self) -> &str { + &self.driver_id + } + fn capabilities(&self) -> Capabilities { + advertised_capabilities() + } + async fn health(&self) -> MemoryHealth { + if self.client.memory_handle().health_check().await { + MemoryHealth::Ready + } else { + MemoryHealth::down("memory store is unavailable") + } + } + fn as_documents(&self) -> Option<&dyn MemoryDocuments> { + Some(self) + } + fn as_ingest(&self) -> Option<&dyn MemoryIngest> { + Some(self) + } + fn as_graph(&self) -> Option<&dyn MemoryGraph> { + Some(self) + } + fn as_goals(&self) -> Option<&dyn MemoryGoals> { + Some(self) + } + fn as_tool_memory(&self) -> Option<&dyn MemoryToolMemory> { + Some(self) + } + fn as_tree(&self) -> Option<&dyn MemoryTree> { + Some(self) + } + fn as_entities(&self) -> Option<&dyn MemoryEntities> { + Some(self) + } + fn as_diff(&self) -> Option<&dyn MemoryDiff> { + // Reachable only when the git-backed snapshot store is compiled in; + // see the `memory-git` feature in this crate's manifest. + #[cfg(feature = "memory-git")] + { + Some(self) + } + #[cfg(not(feature = "memory-git"))] + { + None + } + } + fn as_sources(&self) -> Option<&dyn MemorySourceSink> { + Some(self) + } + fn as_maintenance(&self) -> Option<&dyn MemoryMaintenance> { + Some(self) + } + fn as_people(&self) -> Option<&dyn MemoryPeople> { + Some(self) + } + fn as_chunks(&self) -> Option<&dyn MemoryChunks> { + Some(self) + } + fn as_retrieval(&self) -> Option<&dyn MemoryRetrieval> { + Some(self) + } + fn as_profile(&self) -> Option<&dyn MemoryProfile> { + Some(self) + } + fn as_episodic(&self) -> Option<&dyn MemoryEpisodic> { + Some(self) + } +} + +// ── People ─────────────────────────────────────────────────────────────────── +// +// The conversions below destructure both sides exhaustively rather than +// round-tripping through `Self::cross`. That is deliberate. `cross` is a serde +// value round-trip, so it agrees only while the two crates' field *names* agree +// — and they already do not: the engine's `Interaction` names its timestamp +// `ts` where the contract names it `at`. A round-trip would compile and then +// fail at runtime on the first call. +// +// Destructuring makes the opposite trade: a field added or renamed on either +// side is a compile error here, which is the same rule +// `tinymemory-tinycortex::convert` follows and the same reasoning that governs +// the two copies of the contract itself. + +/// The engine's people store for this module's workspace. +/// +/// `for_workspace` caches per workspace directory, so this is a map lookup +/// after the first call rather than a database open. +fn people_store( + workspace: &std::path::Path, +) -> Result, MemoryError> { + tinycortex::memory::people::store::for_workspace(workspace) + .map_err(|error| MemoryError::Other(anyhow::anyhow!("open people store: {error}"))) +} + +fn handle_to_engine(handle: &PersonHandle) -> tinycortex::memory::people::types::Handle { + use tinycortex::memory::people::types::Handle as EngineHandle; + match handle { + PersonHandle::IMessage(value) => EngineHandle::IMessage(value.clone()), + PersonHandle::Email(value) => EngineHandle::Email(value.clone()), + PersonHandle::DisplayName(value) => EngineHandle::DisplayName(value.clone()), + } +} + +fn handle_to_contract(handle: tinycortex::memory::people::types::Handle) -> PersonHandle { + use tinycortex::memory::people::types::Handle as EngineHandle; + match handle { + EngineHandle::IMessage(value) => PersonHandle::IMessage(value), + EngineHandle::Email(value) => PersonHandle::Email(value), + EngineHandle::DisplayName(value) => PersonHandle::DisplayName(value), + } +} + +fn person_to_contract(person: tinycortex::memory::people::types::Person) -> PersonRecord { + let tinycortex::memory::people::types::Person { + id, + display_name, + primary_email, + primary_phone, + handles, + created_at, + updated_at, + } = person; + PersonRecord { + id: id.to_string(), + display_name, + primary_email, + primary_phone, + handles: handles.into_iter().map(handle_to_contract).collect(), + created_at: created_at.to_rfc3339(), + updated_at: updated_at.to_rfc3339(), + } +} + +fn score_to_contract( + score: tinycortex::memory::people::types::ScoreComponents, + interaction_count: usize, +) -> PersonScore { + let tinycortex::memory::people::types::ScoreComponents { + recency, + frequency, + reciprocity, + depth, + score, + } = score; + PersonScore { + recency, + frequency, + reciprocity, + depth, + score, + interaction_count, + } +} + +/// Parse a caller-supplied person id. +/// +/// `PersonRef` is opaque to the caller by contract, so an unparseable one is a +/// caller mistake — `Invalid`, not `NotFound`. Reporting `NotFound` would tell +/// a caller the id was well-formed but absent, which would send them looking +/// for a deleted person rather than at the id they built. +fn parse_person_id( + person_id: &str, +) -> Result { + person_id + .parse::() + .map(tinycortex::memory::people::types::PersonId) + .map_err(|_| MemoryError::Invalid(format!("malformed person id: {person_id}"))) +} + +#[async_trait] +impl MemoryPeople for TinycortexProvider { + async fn list_people(&self, limit: Option) -> Result, MemoryError> { + let store = people_store(&self.config.workspace_dir)?; + let people = store + .list() + .await + .map_err(|error| Self::other("list people", error))?; + + let ids: Vec<_> = people.iter().map(|person| person.id).collect(); + let interactions = store + .batch_interactions_for(&ids) + .await + .map_err(|error| Self::other("load interactions", error))?; + + let now = Utc::now(); + let mut ranked: Vec = people + .into_iter() + .map(|person| { + let observed = interactions.get(&person.id).map_or(&[][..], Vec::as_slice); + let closeness = tinycortex::memory::people::scorer::score(observed, now); + RankedPerson { + person: person_to_contract(person), + score: score_to_contract(closeness, observed.len()), + } + }) + .collect(); + + // Descending by composite score. `total_cmp` rather than `partial_cmp`: + // a NaN from a degenerate score would make `partial_cmp` return `None`, + // and an ordering that is not total is undefined behaviour's + // well-behaved cousin — `sort_by` may panic or produce garbage order. + ranked.sort_by(|a, b| b.score.score.total_cmp(&a.score.score)); + if let Some(limit) = limit { + ranked.truncate(limit); + } + Ok(ranked) + } + + async fn get_person(&self, person_id: &str) -> Result, MemoryError> { + let store = people_store(&self.config.workspace_dir)?; + let id = parse_person_id(person_id)?; + Ok(store + .get(id) + .await + .map_err(|error| Self::other("get person", error))? + .map(person_to_contract)) + } + + async fn resolve_handle( + &self, + handle: &PersonHandle, + create_if_missing: bool, + ) -> Result, MemoryError> { + let store = people_store(&self.config.workspace_dir)?; + let resolver = tinycortex::memory::people::resolver::HandleResolver::new(&store); + let engine_handle = handle_to_engine(handle); + + if create_if_missing { + let (id, created) = resolver + .resolve_or_create_with_status(&engine_handle) + .await + .map_err(|error| Self::other("resolve or create handle", error))?; + return Ok(Some(ResolvedPerson { + id: id.to_string(), + created, + })); + } + + Ok(resolver + .resolve(&engine_handle) + .await + .map_err(|error| Self::other("resolve handle", error))? + .map(|id| ResolvedPerson { + id: id.to_string(), + created: false, + })) + } + + async fn add_handle_alias( + &self, + person_id: &str, + handle: &PersonHandle, + ) -> Result<(), MemoryError> { + let store = people_store(&self.config.workspace_dir)?; + let id = parse_person_id(person_id)?; + if store + .get(id) + .await + .map_err(|error| Self::other("look up person", error))? + .is_none() + { + return Err(MemoryError::NotFound(format!("person {person_id}"))); + } + store + .add_alias(id, handle_to_engine(handle).canonicalize()) + .await + .map_err(|error| Self::other("add handle alias", error)) + } + + async fn score_person(&self, person_id: &str) -> Result, MemoryError> { + let store = people_store(&self.config.workspace_dir)?; + let id = parse_person_id(person_id)?; + if store + .get(id) + .await + .map_err(|error| Self::other("look up person", error))? + .is_none() + { + return Ok(None); + } + let interactions = store + .interactions_for(id) + .await + .map_err(|error| Self::other("load interactions", error))?; + Ok(Some(score_to_contract( + tinycortex::memory::people::scorer::score(&interactions, Utc::now()), + interactions.len(), + ))) + } + + async fn record_interaction(&self, interaction: &PersonInteraction) -> Result<(), MemoryError> { + let store = people_store(&self.config.workspace_dir)?; + let PersonInteraction { + person_id, + at, + is_outbound, + length, + } = interaction; + let id = parse_person_id(person_id)?; + let ts = chrono::DateTime::parse_from_rfc3339(at) + .map_err(|error| MemoryError::Invalid(format!("malformed interaction time: {error}")))? + .with_timezone(&Utc); + if store + .get(id) + .await + .map_err(|error| Self::other("look up person", error))? + .is_none() + { + return Err(MemoryError::NotFound(format!("person {person_id}"))); + } + store + .record_interaction(tinycortex::memory::people::types::Interaction { + person_id: id, + ts, + is_outbound: *is_outbound, + length: *length, + }) + .await + .map_err(|error| Self::other("record interaction", error)) + } + + async fn seed_from_address_book(&self) -> Result { + let store = people_store(&self.config.workspace_dir)?; + let resolver = tinycortex::memory::people::resolver::HandleResolver::new(&store); + let source = tinycortex::memory::people::address_book::SystemContactsSource; + let (seeded, skipped) = resolver + .seed_from_address_book(&source) + .await + .map_err(|error| Self::other("seed from address book", error))?; + Ok(AddressBookSeedOutcome { seeded, skipped }) + } +} + +// ── Chunks and Retrieval ───────────────────────────────────────────────────── +// +// Both families take the source scope as an **argument** and never read the +// ambient one. `tinymemory_core`'s in-process entry points resolve it from a +// task-local, which the host sets on its own side of the bus — it is simply not +// present in this process. Reading it here would yield `None`, and `None` means +// *unrestricted*, so a per-profile source gate would fail open. That is why the +// `*_scoped` variants exist and why these call them. + +/// Convert a contract scope into the engine's allowlist form. +fn scope_to_engine(scope: Option<&SourceScope>) -> Option> { + scope.map(|scope| scope.allow.iter().cloned().collect()) +} + +#[async_trait] +impl MemoryChunks for TinycortexProvider { + async fn list_chunks( + &self, + query: &ChunkQuery, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + let ChunkQuery { + source_kind, + source_id, + owner, + since_ms, + until_ms, + limit, + offset, + exclude_dropped, + } = query.clone(); + let engine_query = tinymemory_core::store::chunks::ListChunksQuery { + source_kind: source_kind + .map(|kind| Self::cross(&kind, "convert source kind")) + .transpose()?, + source_id, + owner, + since_ms, + until_ms, + limit, + offset, + source_scope: scope_to_engine(scope), + exclude_dropped, + }; + let chunks = blocking(self.config.clone(), "list chunks", move |config| { + tinymemory_core::store::chunks::list_chunks(config, &engine_query) + }) + .await?; + Self::cross(&chunks, "convert chunks") + } + + async fn get_chunk(&self, chunk_id: &str) -> Result, MemoryError> { + let id = chunk_id.to_string(); + let chunk = blocking(self.config.clone(), "get chunk", move |config| { + tinymemory_core::store::chunks::get_chunk(config, &id) + }) + .await?; + match chunk { + Some(chunk) => Ok(Some(Self::cross(&chunk, "convert chunk")?)), + None => Ok(None), + } + } + + async fn chunk_detail(&self, chunk_id: &str) -> Result, MemoryError> { + let id = chunk_id.to_string(); + let detail = blocking(self.config.clone(), "chunk detail", move |config| { + let Some(chunk) = tinymemory_core::store::chunks::get_chunk(config, &id)? else { + return Ok(None); + }; + // The vault read is best-effort: a missing body is reported as + // `None` so the caller can fall back to the row's own content, + // rather than failing the whole detail view over a preview. + let body = tinymemory_core::store::content::read::read_chunk_body(config, &id).ok(); + let has_embedding = + tinymemory_core::store::chunks::get_chunk_embedding(config, &id)?.is_some(); + let lifecycle_status = + tinymemory_core::store::chunks::get_chunk_lifecycle_status(config, &id)?; + let content_path = tinymemory_core::store::chunks::get_chunk_content_path(config, &id)?; + Ok(Some(( + chunk, + body, + has_embedding, + lifecycle_status, + content_path, + ))) + }) + .await?; + + let Some((chunk, body, has_embedding, lifecycle_status, content_path)) = detail else { + return Ok(None); + }; + Ok(Some(ChunkDetail { + chunk: Self::cross(&chunk, "convert chunk")?, + body, + content_path, + lifecycle_status, + has_embedding, + })) + } + + async fn storage_kinds(&self) -> Result, MemoryError> { + Ok(tinymemory_core::store::MemoryKind::ALL + .iter() + .map(|kind| kind.as_str().to_string()) + .collect()) + } + + async fn chunk_embeddings( + &self, + chunk_ids: &[String], + model_signature: &str, + ) -> Result, MemoryError> { + let ids = chunk_ids.to_vec(); + let signature = model_signature.to_string(); + let vectors = blocking( + self.config.clone(), + "load chunk embeddings", + move |config| { + tinymemory_core::store::chunks::get_chunk_embeddings_for_signature_batch( + config, &ids, &signature, + ) + }, + ) + .await?; + // Sorted so the response is deterministic: the engine returns a + // `HashMap`, whose iteration order varies per process and would make an + // otherwise-identical call return a differently-ordered list. + let mut embeddings: Vec = vectors + .into_iter() + .map(|(chunk_id, vector)| ChunkEmbedding { chunk_id, vector }) + .collect(); + embeddings.sort_by(|a, b| a.chunk_id.cmp(&b.chunk_id)); + Ok(embeddings) + } +} + +#[async_trait] +impl MemoryRetrieval for TinycortexProvider { + async fn fast_retrieve( + &self, + query: &str, + options: FastRetrieveQuery, + scope: Option<&SourceScope>, + ) -> Result { + if query.trim().is_empty() { + return Err(MemoryError::Invalid("query must not be empty".to_string())); + } + let engine_options = tinymemory_core::tree::retrieval::FastRetrieveOptions { + limit: options.limit, + max_hops: options.max_hops, + time_window_days: options.time_window_days, + }; + let response = tinymemory_core::tree::retrieval::fast_retrieve_scoped( + &self.config, + query, + engine_options, + scope_to_engine(scope), + ) + .await + .map_err(|error| Self::other("fast retrieve", error))?; + Self::cross(&response, "convert retrieval response") + } + + async fn cover_window( + &self, + window: &CoverWindowQuery, + scope: Option<&SourceScope>, + ) -> Result { + let CoverWindowQuery { + since_ms, + until_ms, + source_id, + source_kind, + limit, + } = window.clone(); + let engine_kind = source_kind + .map(|kind| Self::cross(&kind, "convert source kind")) + .transpose()?; + let response = tinymemory_core::tree::retrieval::cover_window_scoped( + &self.config, + since_ms, + until_ms, + source_id.as_deref(), + engine_kind, + // 0 is the engine's "no caller preference" sentinel, not a request + // for zero rows: `cover_window_scoped` substitutes its own + // DEFAULT_LIMIT for it. Mapping `None` to 0 therefore asks for the + // default, which is what an absent limit means. + limit.unwrap_or(0), + scope_to_engine(scope), + ) + .await + .map_err(|error| Self::other("cover window", error))?; + Self::cross(&response, "convert retrieval response") + } + + async fn retrieve_source( + &self, + query: &SourceRetrievalQuery, + scope: Option<&SourceScope>, + ) -> Result { + let SourceRetrievalQuery { + source_id, + source_kind, + time_window_days, + query: text, + limit, + } = query.clone(); + let engine_kind = source_kind + .map(|kind| Self::cross(&kind, "convert source kind")) + .transpose()?; + let response = tinymemory_core::tree::retrieval::source::query_source_scoped( + &self.config, + tinymemory_core::tree::retrieval::source::SourceQuery { + source_id: source_id.as_deref(), + source_kind: engine_kind, + time_window_days, + query: text.as_deref(), + limit, + }, + scope_to_engine(scope), + ) + .await + .map_err(|error| Self::other("retrieve source", error))?; + Self::cross(&response, "convert retrieval response") + } + + async fn retrieve_children( + &self, + node_id: &str, + max_depth: u32, + query: Option<&str>, + limit: Option, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + let hits = tinymemory_core::tree::retrieval::drill_down::drill_down_scoped( + &self.config, + node_id, + max_depth, + query, + limit, + scope_to_engine(scope), + ) + .await + .map_err(|error| Self::other("drill down", error))?; + Self::cross(&hits, "convert retrieval hits") + } + + async fn retrieve_leaves( + &self, + chunk_ids: &[String], + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + let hits = tinymemory_core::tree::retrieval::fetch::fetch_leaves_scoped( + &self.config, + chunk_ids, + scope_to_engine(scope), + ) + .await + .map_err(|error| Self::other("fetch leaves", error))?; + Self::cross(&hits, "convert retrieval hits") + } + + async fn recall_namespace_scored( + &self, + namespace: &str, + query: &str, + limit: usize, + exclude_session_id: Option<&str>, + ) -> Result, MemoryError> { + let hits = self + .client + .unified_handle() + .query_namespace_hits_excluding_session( + namespace, + query, + u32::try_from(limit).unwrap_or(u32::MAX), + exclude_session_id, + ) + .await + .map_err(|error| Self::other("recall namespace scored", error))?; + Self::cross(&hits, "convert namespace hits") + } + + async fn search_entities( + &self, + query: &str, + kinds: Option<&[String]>, + limit: usize, + ) -> Result, MemoryError> { + // Request kinds are validated, unlike response kinds which pass through + // as an open vocabulary. An unknown filter that silently matched nothing + // would be indistinguishable from a genuine empty result. + let engine_kinds = match kinds { + Some(kinds) => Some( + kinds + .iter() + .map(|kind| { + tinymemory_core::tree::score::extract::EntityKind::parse(kind).map_err( + |_| MemoryError::Invalid(format!("unknown entity kind: {kind}")), + ) + }) + .collect::, MemoryError>>()?, + ), + None => None, + }; + let matches = tinymemory_core::tree::retrieval::search_entities( + &self.config, + query, + engine_kinds, + limit, + ) + .await + .map_err(|error| Self::other("search entities", error))?; + Self::cross(&matches, "convert entity matches") + } +} + +// ── Profile ────────────────────────────────────────────────────────────────── +// +// `ProfileStore`'s methods are synchronous and hold a `parking_lot::Mutex` +// across a SQLite call, so each one goes through `spawn_blocking` rather than +// being awaited on the runtime thread. The store is cheap to obtain — it is a +// handle over the client's connection, not an open — so it is fetched inside +// the blocking closure rather than held across an await. + +fn facet_type_to_engine( + facet_type: FacetType, +) -> tinymemory_core::store::namespace_store::profile::FacetType { + use tinymemory_core::store::namespace_store::profile::FacetType as Engine; + match facet_type { + FacetType::Preference => Engine::Preference, + FacetType::Workflow => Engine::Workflow, + FacetType::Role => Engine::Role, + FacetType::Personality => Engine::Personality, + FacetType::Context => Engine::Context, + } +} + +#[async_trait] +impl MemoryProfile for TinycortexProvider { + async fn list_active_facets(&self) -> Result, MemoryError> { + let client = Arc::clone(&self.client); + let facets = tokio::task::spawn_blocking(move || client.profile_store().list_active()) + .await + .map_err(|e| Self::other("join list_active_facets", e))? + .map_err(|e| Self::other("list_active_facets", e))?; + Self::cross(&facets, "convert facets") + } + + async fn list_all_facets(&self) -> Result, MemoryError> { + let client = Arc::clone(&self.client); + let facets = tokio::task::spawn_blocking(move || client.profile_store().list_all()) + .await + .map_err(|e| Self::other("join list_all_facets", e))? + .map_err(|e| Self::other("list_all_facets", e))?; + Self::cross(&facets, "convert facets") + } + + async fn get_facet(&self, key: &str) -> Result, MemoryError> { + let client = Arc::clone(&self.client); + let key = key.to_string(); + let facet = tokio::task::spawn_blocking(move || client.profile_store().get(&key)) + .await + .map_err(|e| Self::other("join get_facet", e))? + .map_err(|e| Self::other("get_facet", e))?; + match facet { + Some(facet) => Ok(Some(Self::cross(&facet, "convert facet")?)), + None => Ok(None), + } + } + + async fn facets_by_type( + &self, + facet_type: FacetType, + ) -> Result, MemoryError> { + let client = Arc::clone(&self.client); + let engine = facet_type_to_engine(facet_type); + let facets = + tokio::task::spawn_blocking(move || client.profile_store().facets_by_type(&engine)) + .await + .map_err(|e| Self::other("join facets_by_type", e))? + .map_err(|e| Self::other("facets_by_type", e))?; + Self::cross(&facets, "convert facets") + } + + async fn upsert_facet(&self, facet: &ProfileFacet) -> Result<(), MemoryError> { + let client = Arc::clone(&self.client); + let engine: tinymemory_core::store::namespace_store::profile::ProfileFacet = + Self::cross(facet, "convert facet")?; + tokio::task::spawn_blocking(move || client.profile_store().upsert_full(&engine)) + .await + .map_err(|e| Self::other("join upsert_facet", e))? + .map_err(|e| Self::other("upsert_facet", e)) + } + + async fn upsert_provider_facet( + &self, + facet_id: &str, + facet_type: FacetType, + key: &str, + value: &str, + confidence: f64, + segment_id: Option<&str>, + observed_at: f64, + ) -> Result<(), MemoryError> { + let client = Arc::clone(&self.client); + let engine = facet_type_to_engine(facet_type); + let (facet_id, key, value) = (facet_id.to_string(), key.to_string(), value.to_string()); + let segment_id = segment_id.map(str::to_string); + tokio::task::spawn_blocking(move || { + client.profile_store().upsert_provider_facet( + &facet_id, + &engine, + &key, + &value, + confidence, + segment_id.as_deref(), + observed_at, + ) + }) + .await + .map_err(|e| Self::other("join upsert_provider_facet", e))? + .map_err(|e| Self::other("upsert_provider_facet", e)) + } + + async fn set_facet_user_state( + &self, + key: &str, + user_state: UserState, + ) -> Result { + use tinymemory_core::store::namespace_store::profile::UserState as Engine; + let client = Arc::clone(&self.client); + let key = key.to_string(); + let engine = match user_state { + UserState::Auto => Engine::Auto, + UserState::Pinned => Engine::Pinned, + UserState::Forgotten => Engine::Forgotten, + }; + tokio::task::spawn_blocking(move || client.profile_store().set_user_state(&key, engine)) + .await + .map_err(|e| Self::other("join set_facet_user_state", e))? + .map_err(|e| Self::other("set_facet_user_state", e)) + } + + async fn delete_facet(&self, key: &str) -> Result { + let client = Arc::clone(&self.client); + let key = key.to_string(); + tokio::task::spawn_blocking(move || client.profile_store().delete(&key)) + .await + .map_err(|e| Self::other("join delete_facet", e))? + .map_err(|e| Self::other("delete_facet", e)) + } + + async fn delete_facet_by_id(&self, facet_id: &str) -> Result { + let client = Arc::clone(&self.client); + let facet_id = facet_id.to_string(); + tokio::task::spawn_blocking(move || client.profile_store().delete_by_facet_id(&facet_id)) + .await + .map_err(|e| Self::other("join delete_facet_by_id", e))? + .map_err(|e| Self::other("delete_facet_by_id", e)) + } + + async fn drop_facets_below(&self, threshold: f64) -> Result { + let client = Arc::clone(&self.client); + tokio::task::spawn_blocking(move || client.profile_store().drop_below_threshold(threshold)) + .await + .map_err(|e| Self::other("join drop_facets_below", e))? + .map_err(|e| Self::other("drop_facets_below", e)) + } + + async fn workflow_identity_matches(&self, key_pattern: &str, canonical_value: &str) -> bool { + let client = Arc::clone(&self.client); + let (pattern, value) = (key_pattern.to_string(), canonical_value.to_string()); + tokio::task::spawn_blocking(move || { + client + .profile_store() + .skill_identity_matches(&pattern, &value) + }) + .await + // A join failure reads as "no", like every other error on this + // predicate — see the trait docs. But it is logged first: the two + // cases behind it are a cancelled task and a panic inside + // `skill_identity_matches`, and a panic is a defect. Answering a bare + // `false` would make that defect look exactly like a legitimate + // non-match, which is the one reading that guarantees nobody + // investigates it. + .inspect_err(|error| { + log::error!( + "[tinymemory:module] workflow_identity_matches join failed, answering false: \ + {error}" + ); + }) + .unwrap_or(false) + } +} + +/// Episodic capture: the turn-by-turn record and its segment lifecycle. +/// +/// Every method hops to `spawn_blocking` for the same reason the profile family +/// does — these are synchronous `rusqlite` calls behind a `parking_lot::Mutex`, +/// and blocking a tinybus executor thread on a database lock would stall every +/// other call the module is serving. +/// +/// The boundary-detection and summary-composition halves of the archivist are +/// **not** here: they touch no database and are host policy. See the family's +/// contract docs. +#[async_trait] +impl MemoryEpisodic for TinycortexProvider { + async fn insert_turn(&self, turn: &EpisodicTurn) -> Result { + let conn = self.client.profile_conn(); + let entry = tinymemory_core::store::fts5::EpisodicEntry { + id: None, + session_id: turn.session_id.clone(), + timestamp: turn.timestamp, + role: turn.role.clone(), + content: turn.content.clone(), + lesson: turn.lesson.clone(), + tool_calls_json: turn.tool_calls_json.clone(), + // The contract carries this signed because a cost is a plain number + // on the wire; the engine column is unsigned. A negative value is + // not meaningful, so it clamps rather than wrapping. + cost_microdollars: u64::try_from(turn.cost_microdollars).unwrap_or(0), + }; + tokio::task::spawn_blocking(move || { + tinymemory_core::store::fts5::episodic_insert(&conn, &entry) + }) + .await + .map_err(|e| Self::other("join insert_turn", e))? + .map_err(|e| Self::other("insert_turn", e)) + } + + async fn session_turns(&self, session_id: &str) -> Result, MemoryError> { + let conn = self.client.profile_conn(); + let session_id = session_id.to_string(); + let entries = tokio::task::spawn_blocking(move || { + tinymemory_core::store::fts5::episodic_session_entries(&conn, &session_id) + }) + .await + .map_err(|e| Self::other("join session_turns", e))? + .map_err(|e| Self::other("session_turns", e))?; + Ok(entries.into_iter().map(episodic_to_contract).collect()) + } + + async fn open_segment( + &self, + session_id: &str, + ) -> Result, MemoryError> { + let conn = self.client.profile_conn(); + let session_id = session_id.to_string(); + let segment = tokio::task::spawn_blocking(move || { + tinymemory_core::store::segments::open_segment_for_session(&conn, &session_id) + }) + .await + .map_err(|e| Self::other("join open_segment", e))? + .map_err(|e| Self::other("open_segment", e))?; + Ok(segment.map(segment_to_contract)) + } + + async fn create_segment( + &self, + segment_id: &str, + session_id: &str, + namespace: &str, + start_episodic_id: i64, + start_timestamp: f64, + now: f64, + ) -> Result<(), MemoryError> { + let conn = self.client.profile_conn(); + let (segment_id, session_id, namespace) = ( + segment_id.to_string(), + session_id.to_string(), + namespace.to_string(), + ); + tokio::task::spawn_blocking(move || { + tinymemory_core::store::segments::segment_create( + &conn, + &segment_id, + &session_id, + &namespace, + start_episodic_id, + // Per-session seq numbering is the archivist store's, and it is + // not part of this contract; legacy rows carry `None` too. + None, + start_timestamp, + now, + ) + }) + .await + .map_err(|e| Self::other("join create_segment", e))? + .map_err(|e| Self::other("create_segment", e)) + } + + async fn append_turn( + &self, + segment_id: &str, + episodic_id: i64, + timestamp: f64, + now: f64, + ) -> Result<(), MemoryError> { + let conn = self.client.profile_conn(); + let segment_id = segment_id.to_string(); + tokio::task::spawn_blocking(move || { + tinymemory_core::store::segments::segment_append_turn( + &conn, + &segment_id, + episodic_id, + None, + timestamp, + now, + ) + }) + .await + .map_err(|e| Self::other("join append_turn", e))? + .map_err(|e| Self::other("append_turn", e)) + } + + async fn close_segment(&self, segment_id: &str, now: f64) -> Result<(), MemoryError> { + let conn = self.client.profile_conn(); + let segment_id = segment_id.to_string(); + tokio::task::spawn_blocking(move || { + tinymemory_core::store::segments::segment_close(&conn, &segment_id, now) + }) + .await + .map_err(|e| Self::other("join close_segment", e))? + .map_err(|e| Self::other("close_segment", e)) + } + + async fn set_segment_summary( + &self, + segment_id: &str, + summary: &str, + now: f64, + ) -> Result<(), MemoryError> { + let conn = self.client.profile_conn(); + let (segment_id, summary) = (segment_id.to_string(), summary.to_string()); + tokio::task::spawn_blocking(move || { + tinymemory_core::store::segments::segment_set_summary(&conn, &segment_id, &summary, now) + }) + .await + .map_err(|e| Self::other("join set_segment_summary", e))? + .map_err(|e| Self::other("set_segment_summary", e)) + } + + async fn upsert_segment_embedding( + &self, + segment_id: &str, + model_signature: &str, + embedding: &[f32], + created_at: f64, + ) -> Result<(), MemoryError> { + let conn = self.client.profile_conn(); + let (segment_id, model_signature) = (segment_id.to_string(), model_signature.to_string()); + let embedding = embedding.to_vec(); + tokio::task::spawn_blocking(move || { + tinymemory_core::store::segments::segment_embedding_upsert( + &conn, + &segment_id, + &model_signature, + &embedding, + created_at, + ) + }) + .await + .map_err(|e| Self::other("join upsert_segment_embedding", e))? + .map_err(|e| Self::other("upsert_segment_embedding", e)) + } +} + +/// Engine episodic row -> contract turn. +fn episodic_to_contract(entry: tinymemory_core::store::fts5::EpisodicEntry) -> EpisodicTurn { + EpisodicTurn { + id: entry.id, + session_id: entry.session_id, + timestamp: entry.timestamp, + role: entry.role, + content: entry.content, + lesson: entry.lesson, + tool_calls_json: entry.tool_calls_json, + cost_microdollars: i64::try_from(entry.cost_microdollars).unwrap_or(i64::MAX), + } +} + +/// Engine segment row -> contract segment. +/// +/// Written out rather than derived: the engine row carries several fields the +/// contract deliberately does not expose (`topic_keywords`, the seq numbers, +/// `created_at`), and a blanket conversion would quietly start shipping them if +/// the contract ever grew a matching name. +fn segment_to_contract( + segment: tinymemory_core::store::segments::ConversationSegment, +) -> ConversationSegment { + use tinymemory_core::store::segments::SegmentStatus; + ConversationSegment { + segment_id: segment.segment_id, + session_id: segment.session_id, + namespace: segment.namespace, + start_episodic_id: segment.start_episodic_id, + end_episodic_id: segment.end_episodic_id, + start_timestamp: segment.start_timestamp, + end_timestamp: segment.end_timestamp, + turn_count: segment.turn_count, + summary: segment.summary, + embedding: segment.embedding, + open: matches!(segment.status, SegmentStatus::Open), + } +} + +#[cfg(test)] +mod test; diff --git a/adapters/tinycortex/src/engine/test.rs b/adapters/tinycortex/src/engine/test.rs new file mode 100644 index 0000000..a53d9b4 --- /dev/null +++ b/adapters/tinycortex/src/engine/test.rs @@ -0,0 +1,54 @@ +//! Capability honesty for the full engine provider. +//! +//! The point of lifting the optional families here (issue #18 §C3) is that a +//! host filtering its surface from a negotiated capability set gets the whole +//! engine rather than the mandatory third of it. That is only safe if the set +//! is true. +//! +//! These assert the rule directly rather than through a constructed provider. +//! Construction needs a `MemoryClient`, which needs the host's process-global +//! seams (`set_embedding_host` and friends) installed — and a test that installs +//! a process global is order-dependent, which `AGENTS.md` rules out. The +//! provider-level check that `capabilities()` equals the reachable accessors is +//! `audit_provider`, and it runs against a real engine in the conformance suite +//! once a host has wired those seams. + +#![allow(clippy::expect_used, clippy::panic)] + +use tinymemory_api::capabilities::{Capabilities, Capability}; + +use super::advertised_capabilities; + +#[test] +fn the_mandatory_families_are_always_advertised() { + let caps = advertised_capabilities(); + for mandatory in Capability::MANDATORY { + assert!( + caps.contains(mandatory), + "`{}` must be advertised in every build", + mandatory.as_str() + ); + } +} + +#[cfg(feature = "memory-git")] +#[test] +fn the_full_engine_advertises_every_family_with_memory_git() { + // The lift's headline: this adapter used to advertise three families. + assert_eq!(advertised_capabilities(), Capabilities::all()); + assert!(advertised_capabilities().contains(Capability::Diff)); +} + +#[cfg(not(feature = "memory-git"))] +#[test] +fn diff_is_withheld_when_the_snapshot_store_is_compiled_out() { + // The gate has to reach the advertisement, not just the accessor. A build + // that advertised `Diff` here would fail `audit_provider` — which is how + // that audit earns its place. + let caps = advertised_capabilities(); + assert!(!caps.contains(Capability::Diff)); + // Everything else the engine serves is still advertised: withholding one + // family must not quietly withhold the rest. + assert_eq!(caps, Capabilities::all().without(Capability::Diff)); + assert_eq!(caps.len(), Capabilities::all().len() - 1); +} diff --git a/adapters/tinycortex/src/lib.rs b/adapters/tinycortex/src/lib.rs index 078e747..ea9ead8 100644 --- a/adapters/tinycortex/src/lib.rs +++ b/adapters/tinycortex/src/lib.rs @@ -13,26 +13,40 @@ //! - [`TinycortexMemory`] — wraps any TinyCortex [`tinycortex::memory::Memory`] //! backend as a TinyMemory //! [`Memory`](tinymemory_api::traits::Memory). -//! - [`provider`] — the one call that turns a TinyCortex backend into a bound -//! driver, by pairing [`TinycortexMemory`] with -//! [`MemoryTraitProvider`]. +//! - [`provider`] — the one call that turns a TinyCortex backend into a +//! mandatory-only driver, by pairing [`TinycortexMemory`] with +//! [`MemoryTraitProvider`]. Enough when a host wants store, recall and +//! export and nothing else. +//! - [`engine`] — [`TinycortexProvider`](engine::TinycortexProvider), the whole +//! engine behind the contract: trees, chunks, entities, the graph, goals, +//! tool-memory, ingestion, sources, maintenance, people, retrieval, profile, +//! episodic, and — with `memory-git` — the diff ledger. //! -//! ## Scope: the mandatory three, not the whole engine +//! ## Two drivers, and why both //! -//! A driver built here advertises Core, Recall and Portability. TinyCortex can -//! do far more — trees, chunks, entities, a diff ledger — but those families -//! are reached through engine entry points that need a host's configuration, -//! embedding compute and job queue, none of which this crate has. A host that -//! provides them implements the optional families itself and delegates only the -//! mandatory three here. +//! [`provider`] advertises Core, Recall and Portability. That used to be the +//! only thing here, and it was the reason anything wanting a summary tree or a +//! diff ledger reached past the contract to the engine directly: the families +//! existed, but not through `MemoryProvider`. Issue #18 §C3 lifted those +//! implementations here from `tinymemory-module`, which had grown them because +//! it needed them and nowhere else had them. //! -//! Advertising only what is reachable is deliberate, not a shortcut: a driver -//! whose capability set overstates its accessors fails +//! [`engine::TinycortexProvider`] needs what they need — a workspace, a host +//! configuration, and a `MemoryClient` — so it is the heavier of the two, and a +//! host that has none of that still has [`provider`]. +//! +//! ## Capability honesty +//! +//! Both advertise exactly what they reach. That is deliberate, not a shortcut: +//! a driver whose capability set overstates its accessors fails //! [`audit_provider`](tinymemory_api::provider::audit_provider), and a host that //! filtered its RPC surface from an overstated set would register methods that -//! answer errors. +//! answer errors. It is also why the `memory-git` feature reaches +//! [`engine::advertised_capabilities`] and not just the accessor — a build +//! without the git-backed snapshot store must not claim a diff ledger. pub mod convert; +pub mod engine; mod memory; pub use memory::TinycortexMemory; diff --git a/api/Cargo.toml b/api/Cargo.toml index 673c26e..3d5f32d 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -6,6 +6,7 @@ name = "tinymemory-api" publish = false version = "0.1.1" edition = "2021" +rust-version = "1.96" license = "MIT" repository = "https://github.com/tinyhumansai/tinymemory" description = "Stable public contracts for the TinyMemory memory system" diff --git a/api/src/host/config.rs b/api/src/host/config.rs index af3d79f..22920a0 100644 --- a/api/src/host/config.rs +++ b/api/src/host/config.rs @@ -125,6 +125,25 @@ pub trait MemoryHostConfig: Send + Sync + std::fmt::Debug { /// `provider:model` routing string for the memory workload, if pinned. fn memory_provider(&self) -> Option<&str>; + /// The memory **engine** this host selects, when its configuration names + /// one — `tinycortex`, `supermemory`, `mem0`, `cognee`, `null`. + /// + /// Deliberately distinct from [`Self::memory_provider`], which despite the + /// name is a `provider:model` routing string for the memory *workload* — + /// which language model does summarisation and entity extraction. That is a + /// different axis from which store the memory lives in, and conflating them + /// would let a model change repoint a company's storage. + /// + /// `None` means "the host's default", which the host resolves rather than + /// this trait: the driver registry admits a reserved embedded id with no + /// configuration entry precisely so an unconfigured host still binds + /// something instead of failing to start. + /// + /// Defaulted so adding it breaks no existing implementation. + fn memory_driver(&self) -> Option<&str> { + None + } + /// The local model id for a workload, when that workload is routed to /// Ollama (`"ollama:"`). `None` for cloud or unset workloads. /// diff --git a/api/src/host/test_support.rs b/api/src/host/test_support.rs index 486798f..a7e68f0 100644 --- a/api/src/host/test_support.rs +++ b/api/src/host/test_support.rs @@ -45,6 +45,8 @@ pub struct TestHostConfig { pub embeddings_provider: Option, /// See [`MemoryHostConfig::memory_provider`]. pub memory_provider: Option, + /// See [`MemoryHostConfig::memory_driver`]. `None` selects the host default. + pub memory_driver: Option, /// See [`MemoryHostConfig::api_url`]. pub api_url: Option, /// See [`MemoryHostConfig::default_model`]. @@ -113,6 +115,10 @@ impl MemoryHostConfig for TestHostConfig { self.memory_provider.as_deref() } + fn memory_driver(&self) -> Option<&str> { + self.memory_driver.as_deref() + } + fn workload_local_model(&self, workload: &str) -> Option { let raw = match workload { "memory" => self.memory_provider.as_deref(), diff --git a/conformance/Cargo.toml b/conformance/Cargo.toml new file mode 100644 index 0000000..809620e --- /dev/null +++ b/conformance/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "tinymemory-conformance" +publish = false +version = "0.1.0" +edition = "2021" +rust-version = "1.96" +license = "MIT" +description = "Behavioural conformance suite every MemoryProvider driver must pass" +repository = "https://github.com/tinyhumansai/tinymemory" + +[dependencies] +# The contract under test. This crate deliberately depends on NOTHING else of +# substance: a conformance suite that pulled in an engine would be unable to +# prove that a driver is interchangeable, because it would already have chosen +# one. In particular it must not reach `tinymemory-core`, which links a bundled +# SQLite and the embedded engine unconditionally (issue #18 §D). +tinymemory-api = { path = "../api" } +# `MemoryProvider` and its families are object-safe async traits. +async-trait = "0.1" +# `ExportRecord::payload` is a `serde_json::Value`, so the portability +# assertions have to construct and compare one. +serde_json = "1" +# The reference driver maps a poisoned lock onto `MemoryError::Other`, which is +# `#[from] anyhow::Error`. +anyhow = "1" + +[dev-dependencies] +# The suite's own tests drive it against the reference drivers. +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } + +[lints.rust] +unsafe_code = "forbid" +missing_docs = "warn" +unreachable_pub = "warn" + +[lints.clippy] +all = { level = "warn", priority = -1 } +unwrap_used = "warn" +expect_used = "warn" diff --git a/conformance/src/lib.rs b/conformance/src/lib.rs new file mode 100644 index 0000000..7bfca70 --- /dev/null +++ b/conformance/src/lib.rs @@ -0,0 +1,51 @@ +//! Behavioural conformance for `MemoryProvider` drivers. +//! +//! TinyMemory's premise is that an engine can be swapped without the host +//! learning anything new. [`audit_provider`](tinymemory_api::provider::audit_provider) +//! checks that a driver's advertised capabilities match its reachable +//! accessors, which proves the *shape* is honest. Nothing checked that two +//! drivers answer the same question the same way — and that is the claim the +//! premise actually rests on. +//! +//! This crate is that check. Hand [`assert_provider`] any bound driver and it +//! drives the contract: the mandatory three families, upsert semantics on +//! `(namespace, key)`, namespace isolation, provenance preservation, recall +//! limits, export pagination, and import round-tripping. +//! +//! ```no_run +//! use std::sync::Arc; +//! use tinymemory_conformance::{assert_provider, InMemoryProvider}; +//! +//! # async fn run() { +//! assert_provider(Arc::new(InMemoryProvider::new())).await; +//! # } +//! ``` +//! +//! # What it deliberately does not depend on +//! +//! Only `tinymemory-api`. A conformance suite that pulled in an engine could +//! not prove interchangeability, because it would already have chosen one — and +//! reaching `tinymemory-core` would drag in a bundled SQLite and the embedded +//! engine besides (issue #18 §D). +//! +//! # Provenance is the sharp one +//! +//! [`assert_taint_is_preserved`] is not a formality. A driver that reads back +//! `Internal` for content stored as `ExternalSync` has laundered external +//! content into internal-trust content, and every policy gate keyed on taint is +//! then silently wrong. That failure is invisible until something acts on it. + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +pub mod reference; +pub mod suite; + +pub use reference::{InMemoryProvider, REFERENCE_DRIVER_ID}; +pub use suite::{ + assert_awkward_content_round_trips, assert_capability_audit, assert_export_cursor_terminates, + assert_export_import_round_trip, assert_forget_is_idempotent, assert_list_filters_narrow, + assert_namespaces_are_isolated, assert_provider, assert_recall_respects_limit_and_namespace, + assert_store_get_round_trip, assert_taint_is_preserved, + assert_upsert_replaces_rather_than_duplicates, +}; diff --git a/conformance/src/reference/mod.rs b/conformance/src/reference/mod.rs new file mode 100644 index 0000000..16c70ce --- /dev/null +++ b/conformance/src/reference/mod.rs @@ -0,0 +1,304 @@ +//! An in-memory reference driver. +//! +//! This is the driver the suite is calibrated against: the simplest thing that +//! upholds the contract, with no storage engine, no network, and no +//! configuration. It exists for two reasons. +//! +//! First, a conformance suite needs a known-good subject. An assertion that +//! only ever runs against real engines cannot distinguish "the engine is wrong" +//! from "the assertion is wrong"; running it against a driver whose behaviour is +//! obvious by inspection separates those. +//! +//! Second, it documents the contract by example. Everything here is the +//! minimum a driver must do — the `(namespace, key)` upsert, the fail-closed +//! taint handling, the cursor that terminates on `None` rather than on an empty +//! page — so a new engine author has something short to read. +//! +//! It advertises exactly the three mandatory families and leaves every optional +//! accessor at `None`, which is the honest answer for a store with no tree, no +//! graph, and no ingestion pipeline. + +use std::collections::BTreeMap; +use std::sync::Mutex; + +use async_trait::async_trait; +use tinymemory_api::capabilities::Capabilities; +use tinymemory_api::error::MemoryError; +use tinymemory_api::health::MemoryHealth; +use tinymemory_api::provider::{ + ExportPage, ExportRecord, ImportOutcome, MemoryCore, MemoryPortability, MemoryProvider, + MemoryRecall, SourceScope, +}; +use tinymemory_api::recall::OwnedRecallOpts; +use tinymemory_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; + +/// The driver id this reference binds under. +pub const REFERENCE_DRIVER_ID: &str = "reference"; + +/// How many records one [`MemoryPortability::export_page`] call returns when the +/// caller asks for more than this. Deliberately small so the suite's pagination +/// assertion has something to paginate over without building a large fixture. +const MAX_PAGE: usize = 64; + +/// The row map, keyed by the `(namespace, key)` pair the contract upserts on. +type Rows = BTreeMap<(String, String), MemoryEntry>; + +/// A held lock over [`Rows`]. +type RowGuard<'a> = std::sync::MutexGuard<'a, Rows>; + +/// An in-memory [`MemoryProvider`], keyed exactly as the contract specifies. +#[derive(Debug, Default)] +pub struct InMemoryProvider { + rows: Mutex, +} + +impl InMemoryProvider { + /// Builds an empty reference driver. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Locks the row map, mapping a poisoned lock onto a contract error. + /// + /// A poisoned lock means a previous caller panicked mid-write. Returning an + /// error rather than propagating the panic keeps the driver's failure mode + /// inside the contract, which is what the suite asserts of every driver. + fn rows(&self) -> Result, MemoryError> { + self.rows + .lock() + .map_err(|_| MemoryError::Other(anyhow_poisoned())) + } +} + +/// The one place this crate builds an `anyhow::Error`, so the dependency stays +/// visible rather than scattered. +fn anyhow_poisoned() -> anyhow::Error { + anyhow::anyhow!("reference driver lock poisoned by a panicking caller") +} + +#[async_trait] +impl MemoryCore for InMemoryProvider { + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + // Upsert on `(namespace, key)` — the contract's words. A second store at + // the same pair replaces content, category and session, and does not + // create a duplicate. + self.rows()?.insert( + (namespace.to_string(), key.to_string()), + MemoryEntry { + id: format!("{namespace}::{key}"), + key: key.to_string(), + content: content.to_string(), + namespace: Some(namespace.to_string()), + category, + timestamp: "1970-01-01T00:00:00Z".to_string(), + session_id: session_id.map(str::to_owned), + score: None, + // Persisted as given. A driver that re-stamped this would + // launder external content into internal-trust content, which + // is the failure the parameter exists to prevent. + taint, + }, + ); + Ok(()) + } + + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { + Ok(self + .rows()? + .get(&(namespace.to_string(), key.to_string())) + .cloned()) + } + + async fn forget(&self, namespace: &str, key: &str) -> Result { + Ok(self + .rows()? + .remove(&(namespace.to_string(), key.to_string())) + .is_some()) + } + + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError> { + Ok(self + .rows()? + .values() + .filter(|e| namespace.is_none_or(|ns| e.namespace.as_deref() == Some(ns))) + .filter(|e| category.is_none_or(|c| &e.category == c)) + .filter(|e| session_id.is_none_or(|s| e.session_id.as_deref() == Some(s))) + .cloned() + .collect()) + } + + async fn namespaces(&self) -> Result, MemoryError> { + let rows = self.rows()?; + let mut counts: BTreeMap = BTreeMap::new(); + for entry in rows.values() { + if let Some(ns) = entry.namespace.as_deref() { + *counts.entry(ns.to_string()).or_default() += 1; + } + } + Ok(counts + .into_iter() + .map(|(namespace, count)| NamespaceSummary { + namespace, + count, + last_updated: None, + }) + .collect()) + } +} + +#[async_trait] +impl MemoryRecall for InMemoryProvider { + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + _scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + let needle = query.to_lowercase(); + Ok(self + .rows()? + .values() + .filter(|e| { + opts.namespace + .as_deref() + .is_none_or(|ns| e.namespace.as_deref() == Some(ns)) + }) + .filter(|e| e.content.to_lowercase().contains(&needle)) + .take(limit) + .cloned() + .collect()) + } +} + +#[async_trait] +impl MemoryPortability for InMemoryProvider { + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + // The cursor is an offset rendered as a decimal string. A cursor this + // driver did not issue is `Invalid`, not a silent restart from zero — + // silently restarting would make a resumed export duplicate everything + // it had already written. + let offset: usize = match cursor { + None => 0, + Some(raw) => raw + .parse() + .map_err(|_| MemoryError::Invalid(format!("unknown export cursor: {raw}")))?, + }; + let rows = self.rows()?; + let take = limit.clamp(1, MAX_PAGE); + let records: Vec = rows + .values() + .skip(offset) + .take(take) + .map(|e| ExportRecord { + kind: "entry".to_string(), + id: e.id.clone(), + namespace: e.namespace.clone(), + taint: e.taint, + payload: serde_json::json!({ + "key": e.key, + "content": e.content, + "category": e.category.to_string(), + "session_id": e.session_id, + }), + }) + .collect(); + let consumed = offset + records.len(); + // `None` terminates, not an empty page — the contract is explicit that + // an empty `records` is not the terminator. + let next_cursor = (consumed < rows.len()).then(|| consumed.to_string()); + Ok(ExportPage { + records, + next_cursor, + }) + } + + async fn import_records( + &self, + records: Vec, + ) -> Result { + let mut outcome = ImportOutcome::default(); + for record in records { + let (Some(namespace), Some(key)) = ( + record.namespace.clone(), + record + .payload + .get("key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned), + ) else { + // Per-record rejection is reported, not returned as an error: a + // migration must not abort a whole restore over one bad row. + outcome.failed += 1; + outcome + .errors + .push(format!("record {} lacks a namespace or key", record.id)); + continue; + }; + let content = record + .payload + .get("content") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(); + let category = record + .payload + .get("category") + .and_then(serde_json::Value::as_str) + .and_then(|c| c.parse().ok()) + .unwrap_or(MemoryCategory::Core); + let session_id = record + .payload + .get("session_id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); + // `record.taint` verbatim — see the note on `store`. + self.store( + &namespace, + &key, + &content, + category, + session_id.as_deref(), + record.taint, + ) + .await?; + outcome.imported += 1; + } + Ok(outcome) + } +} + +#[async_trait] +impl MemoryProvider for InMemoryProvider { + fn driver_id(&self) -> &str { + REFERENCE_DRIVER_ID + } + + fn capabilities(&self) -> Capabilities { + // Exactly what is reachable. Advertising more would fail + // `audit_provider`, which is itself one of the suite's assertions. + Capabilities::mandatory() + } + + async fn health(&self) -> MemoryHealth { + MemoryHealth::Ready + } +} diff --git a/conformance/src/suite/mod.rs b/conformance/src/suite/mod.rs new file mode 100644 index 0000000..194770c --- /dev/null +++ b/conformance/src/suite/mod.rs @@ -0,0 +1,643 @@ +//! The behavioural assertions every driver must satisfy. +//! +//! [`assert_provider`] is the entry point: hand it any bound +//! [`MemoryProvider`] and it drives the contract. Each sub-assertion is also +//! public, so a driver that is mid-implementation can run the parts it claims +//! to support and get a useful failure rather than an unrelated one. +//! +//! # What this is for +//! +//! `audit_provider` already checks that a driver's advertised capabilities +//! match its reachable accessors. That is a structural check: it proves the +//! shape is honest, not that the behaviour is. Nothing before this module +//! checked that two drivers answer the same question the same way, which is +//! precisely the claim "swap the engine" rests on. +//! +//! # Conventions +//! +//! Every assertion namespaces its fixtures under a unique prefix and cleans up +//! after itself, so the suite can run against a driver that already holds data +//! and against a shared live service. Assertions panic with a message naming +//! the driver, because a conformance failure is a bug report and the driver id +//! is the first thing its author needs. + +use std::sync::Arc; + +use tinymemory_api::capabilities::Capability; +use tinymemory_api::provider::{audit_provider, ExportRecord, MemoryProvider}; +use tinymemory_api::recall::OwnedRecallOpts; +use tinymemory_api::types::{MemoryCategory, MemoryTaint}; + +/// Runs every assertion in the suite. +/// +/// # Panics +/// +/// Panics on the first violation, naming the driver and what it did instead. +pub async fn assert_provider(provider: Arc) { + let p = provider.as_ref(); + + // Every driver, retaining or not. + assert_capability_audit(p); + assert_forget_is_idempotent(p).await; + assert_namespaces_are_isolated(p).await; + assert_export_cursor_terminates(p).await; + + // The contract permits a driver that accepts writes and discards them — + // `NullMemoryProvider` is exactly that, and it is a legitimate binding for a + // deployment that wants the ports wired and nothing retained. There is no + // capability that declares it, so the suite probes for it rather than + // assuming, and reports which half it ran. + // + // This is deliberately a probe and not a flag the caller passes: a driver + // that *intends* to retain and silently does not is the failure mode worth + // catching, and a caller-supplied flag would let it through. + if !retains_writes(p).await { + return; + } + + assert_store_get_round_trip(p).await; + assert_upsert_replaces_rather_than_duplicates(p).await; + assert_list_filters_narrow(p).await; + assert_taint_is_preserved(p).await; + assert_recall_respects_limit_and_namespace(p).await; + assert_export_import_round_trip(p).await; + assert_awkward_content_round_trips(p).await; +} + +/// Whether this driver reads back what it stores. +/// +/// `false` means `/dev/null` semantics, which the contract allows. The storage +/// assertions are vacuous for such a driver and [`assert_provider`] skips them; +/// the contract-shape assertions still apply and are not skipped. +/// +/// # Panics +/// +/// Panics if the probe itself errors — accepting a write and then failing the +/// read is a fault, distinct from accepting a write and discarding it. +pub async fn retains_writes(provider: &dyn MemoryProvider) -> bool { + let who = provider.driver_id(); + let ns = ns(provider, "probe"); + provider + .store( + &ns, + "probe", + "probe", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed during the retention probe: {e}")); + let seen = provider + .get(&ns, "probe") + .await + .unwrap_or_else(|e| panic!("{who}: get failed during the retention probe: {e}")) + .is_some(); + cleanup(provider, &ns, &["probe"]).await; + seen +} + +/// The advertised capability set equals the reachable one. +/// +/// # Panics +/// +/// Panics when a driver advertises a family it cannot serve, or serves one it +/// does not advertise. +pub fn assert_capability_audit(provider: &dyn MemoryProvider) { + if let Err(audit) = audit_provider(provider) { + panic!( + "driver `{}` failed its capability audit: {audit}", + provider.driver_id() + ); + } + // The three mandatory families are not optional, whatever else is claimed. + let caps = provider.capabilities(); + for mandatory in Capability::MANDATORY { + assert!( + caps.contains(mandatory), + "driver `{}` does not advertise the mandatory `{}` family", + provider.driver_id(), + mandatory.as_str() + ); + } +} + +/// A stored entry comes back with its fields intact. +/// +/// # Panics +/// +/// Panics on any field that does not survive the round trip. +pub async fn assert_store_get_round_trip(provider: &dyn MemoryProvider) { + let ns = ns(provider, "round-trip"); + provider + .store( + &ns, + "k1", + "the quick brown fox", + MemoryCategory::Core, + Some("session-1"), + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{}: store failed: {e}", provider.driver_id())); + + let got = provider + .get(&ns, "k1") + .await + .unwrap_or_else(|e| panic!("{}: get failed: {e}", provider.driver_id())) + .unwrap_or_else(|| panic!("{}: stored entry was not returned", provider.driver_id())); + + let who = provider.driver_id(); + assert_eq!(got.key, "k1", "{who}: key not preserved"); + assert_eq!( + got.content, "the quick brown fox", + "{who}: content not preserved" + ); + assert_eq!( + got.namespace.as_deref(), + Some(ns.as_str()), + "{who}: namespace not preserved" + ); + assert_eq!( + got.category, + MemoryCategory::Core, + "{who}: category not preserved" + ); + assert_eq!( + got.session_id.as_deref(), + Some("session-1"), + "{who}: session not preserved" + ); + + // A key that was never stored is `Ok(None)`, never an error. + let missing = provider + .get(&ns, "never-stored") + .await + .unwrap_or_else(|e| panic!("{who}: get of a missing key errored instead of Ok(None): {e}")); + assert!( + missing.is_none(), + "{who}: get returned an entry for a key never stored" + ); + + cleanup(provider, &ns, &["k1"]).await; +} + +/// Storing twice at one `(namespace, key)` replaces rather than duplicates. +/// +/// # Panics +/// +/// Panics when the second store creates a second row or fails to replace. +pub async fn assert_upsert_replaces_rather_than_duplicates(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let ns = ns(provider, "upsert"); + for content in ["first", "second"] { + provider + .store( + &ns, + "same-key", + content, + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + } + let listed = provider + .list(Some(&ns), None, None) + .await + .unwrap_or_else(|e| panic!("{who}: list failed: {e}")); + assert_eq!( + listed.len(), + 1, + "{who}: a re-store at the same key duplicated the row" + ); + assert_eq!( + listed[0].content, "second", + "{who}: the second store did not replace the first" + ); + cleanup(provider, &ns, &["same-key"]).await; +} + +/// `forget` reports whether the entry existed and is safe to call twice. +/// +/// # Panics +/// +/// Panics when a repeat `forget` errors or misreports. +pub async fn assert_forget_is_idempotent(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let ns = ns(provider, "forget"); + provider + .store( + &ns, + "k", + "v", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + + let first = provider + .forget(&ns, "k") + .await + .unwrap_or_else(|e| panic!("{who}: forget failed: {e}")); + let second = provider + .forget(&ns, "k") + .await + .unwrap_or_else(|e| panic!("{who}: repeat forget errored instead of Ok(false): {e}")); + + // A driver that discards writes (the `null` reference) legitimately reports + // `false` both times; what no driver may do is report `true` for an entry it + // does not hold. + assert!( + !second, + "{who}: forget reported true for an already-forgotten key" + ); + if first { + let gone = provider + .get(&ns, "k") + .await + .unwrap_or_else(|e| panic!("{who}: get failed: {e}")); + assert!( + gone.is_none(), + "{who}: forget reported true but the entry is still readable" + ); + } +} + +/// One namespace's entries do not appear in another's. +/// +/// # Panics +/// +/// Panics when a namespace filter leaks an entry from a sibling namespace. +pub async fn assert_namespaces_are_isolated(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let (a, b) = (ns(provider, "iso-a"), ns(provider, "iso-b")); + provider + .store( + &a, + "k", + "belongs to a", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + + let from_b = provider + .list(Some(&b), None, None) + .await + .unwrap_or_else(|e| panic!("{who}: list failed: {e}")); + assert!( + from_b.is_empty(), + "{who}: listing namespace b returned entries from a: {from_b:?}" + ); + + let get_b = provider + .get(&b, "k") + .await + .unwrap_or_else(|e| panic!("{who}: get failed: {e}")); + assert!( + get_b.is_none(), + "{who}: the same key in a sibling namespace resolved to a's entry" + ); + + cleanup(provider, &a, &["k"]).await; +} + +/// Each `list` filter narrows, and `None` everywhere narrows nothing. +/// +/// # Panics +/// +/// Panics when a filter fails to narrow or narrows the wrong rows. +pub async fn assert_list_filters_narrow(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let ns = ns(provider, "filters"); + provider + .store( + &ns, + "core-a", + "x", + MemoryCategory::Core, + Some("s1"), + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + provider + .store( + &ns, + "daily-b", + "y", + MemoryCategory::Daily, + Some("s2"), + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + + let all = provider + .list(Some(&ns), None, None) + .await + .unwrap_or_else(|e| panic!("{who}: list failed: {e}")); + assert_eq!(all.len(), 2, "{who}: expected both entries with no filter"); + + let by_category = provider + .list(Some(&ns), Some(&MemoryCategory::Core), None) + .await + .unwrap_or_else(|e| panic!("{who}: list failed: {e}")); + assert_eq!( + by_category.len(), + 1, + "{who}: the category filter did not narrow" + ); + assert_eq!( + by_category[0].key, "core-a", + "{who}: the category filter kept the wrong row" + ); + + let by_session = provider + .list(Some(&ns), None, Some("s2")) + .await + .unwrap_or_else(|e| panic!("{who}: list failed: {e}")); + assert_eq!( + by_session.len(), + 1, + "{who}: the session filter did not narrow" + ); + assert_eq!( + by_session[0].key, "daily-b", + "{who}: the session filter kept the wrong row" + ); + + let summaries = provider + .namespaces() + .await + .unwrap_or_else(|e| panic!("{who}: namespaces failed: {e}")); + let mine = summaries.iter().find(|s| s.namespace == ns); + if let Some(summary) = mine { + assert_eq!(summary.count, 2, "{who}: namespace summary miscounted"); + } + + cleanup(provider, &ns, &["core-a", "daily-b"]).await; +} + +/// Provenance survives a store, and is not re-stamped. +/// +/// This is the security-relevant one. A driver that returns `Internal` for +/// content stored as `ExternalSync` has laundered external content into +/// internal-trust content, and every downstream policy gate keyed on taint is +/// then wrong. +/// +/// # Panics +/// +/// Panics when taint does not survive. +pub async fn assert_taint_is_preserved(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let ns = ns(provider, "taint"); + for (key, taint) in [ + ("internal", MemoryTaint::Internal), + ("external", MemoryTaint::ExternalSync), + ] { + provider + .store(&ns, key, "content", MemoryCategory::Core, None, taint) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + if let Some(got) = provider + .get(&ns, key) + .await + .unwrap_or_else(|e| panic!("{who}: get failed: {e}")) + { + assert_eq!( + got.taint, taint, + "{who}: taint was re-stamped on `{key}` — stored {taint:?}, read back {:?}", + got.taint + ); + } + } + cleanup(provider, &ns, &["internal", "external"]).await; +} + +/// `recall` honours its limit and its namespace filter. +/// +/// # Panics +/// +/// Panics when recall exceeds the limit or crosses a namespace. +pub async fn assert_recall_respects_limit_and_namespace(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let (mine, theirs) = (ns(provider, "recall-a"), ns(provider, "recall-b")); + let keys = ["r1", "r2", "r3"]; + for key in keys { + provider + .store( + &mine, + key, + "shared needle text", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + } + provider + .store( + &theirs, + "other", + "shared needle text", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + + let opts = OwnedRecallOpts { + namespace: Some(mine.clone()), + ..Default::default() + }; + let hits = provider + .recall("needle", 2, &opts, None) + .await + .unwrap_or_else(|e| panic!("{who}: recall failed: {e}")); + assert!( + hits.len() <= 2, + "{who}: recall returned {} hits for a limit of 2", + hits.len() + ); + for hit in &hits { + assert_eq!( + hit.namespace.as_deref(), + Some(mine.as_str()), + "{who}: recall crossed a namespace boundary" + ); + } + + cleanup(provider, &mine, &keys).await; + cleanup(provider, &theirs, &["other"]).await; +} + +/// Exported records re-import with their taint intact. +/// +/// # Panics +/// +/// Panics when a round trip loses a record or its provenance. +pub async fn assert_export_import_round_trip(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let ns = ns(provider, "portability"); + provider + .store( + &ns, + "p1", + "portable", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + + let mut mine: Vec = Vec::new(); + let mut cursor: Option = None; + // Bounded: a driver whose cursor never terminates is a hang, and a hang in + // a conformance suite reads as an infrastructure problem rather than a bug. + for _ in 0..64 { + let page = provider + .export_page(cursor.as_deref(), 32) + .await + .unwrap_or_else(|e| panic!("{who}: export_page failed: {e}")); + mine.extend( + page.records + .iter() + .filter(|r| r.namespace.as_deref() == Some(ns.as_str())) + .cloned(), + ); + match page.next_cursor { + Some(next) => cursor = Some(next), + None => break, + } + } + assert!( + !mine.is_empty(), + "{who}: a stored entry did not appear in any export page" + ); + + let exported = mine + .iter() + .find(|r| r.taint == MemoryTaint::ExternalSync) + .unwrap_or_else(|| panic!("{who}: export dropped the record's ExternalSync taint")); + assert_eq!(exported.taint, MemoryTaint::ExternalSync); + + provider.forget(&ns, "p1").await.ok(); + let outcome = provider + .import_records(mine.clone()) + .await + .unwrap_or_else(|e| panic!("{who}: import failed: {e}")); + assert_eq!( + outcome.failed, 0, + "{who}: import rejected its own export: {:?}", + outcome.errors + ); + if outcome.failed > 0 { + assert!( + !outcome.errors.is_empty(), + "{who}: reported failures with no diagnosable reason" + ); + } + + if let Some(back) = provider.get(&ns, "p1").await.unwrap_or(None) { + assert_eq!( + back.taint, + MemoryTaint::ExternalSync, + "{who}: import re-stamped provenance instead of persisting what it was given" + ); + } + cleanup(provider, &ns, &["p1"]).await; +} + +/// The export cursor terminates on `None`, not on an empty page. +/// +/// # Panics +/// +/// Panics when a driver signals completion with an empty page while still +/// handing back a cursor, or rejects nothing for a cursor it never issued. +pub async fn assert_export_cursor_terminates(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let page = provider + .export_page(None, 8) + .await + .unwrap_or_else(|e| panic!("{who}: export_page failed: {e}")); + if page.records.is_empty() { + assert!( + page.next_cursor.is_none(), + "{who}: an empty page handed back a cursor — a caller following it cannot terminate" + ); + } + // A cursor this driver never issued must be refused rather than silently + // restarting the export from the beginning, which would duplicate rows. + let bogus = provider.export_page(Some("!not-a-cursor!"), 8).await; + if let Ok(page) = bogus { + assert!( + page.records.is_empty(), + "{who}: an unrecognised cursor returned records instead of being refused" + ); + } +} + +/// Unicode, empty, and oversized content survive a round trip. +/// +/// # Panics +/// +/// Panics when any of them is mangled. +pub async fn assert_awkward_content_round_trips(provider: &dyn MemoryProvider) { + let who = provider.driver_id(); + let ns = ns(provider, "awkward"); + let cases: [(&str, String); 4] = [ + ("unicode", "héllo — 👋 まいど".to_string()), + ("empty", String::new()), + ("large", "x".repeat(64 * 1024)), + ("newlines", "a\nb\r\nc\0d".to_string()), + ]; + for (key, content) in &cases { + provider + .store( + &ns, + key, + content, + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store of `{key}` failed: {e}")); + if let Some(got) = provider + .get(&ns, key) + .await + .unwrap_or_else(|e| panic!("{who}: get of `{key}` failed: {e}")) + { + assert_eq!(&got.content, content, "{who}: `{key}` content was mangled"); + } + } + let keys: Vec<&str> = cases.iter().map(|(k, _)| *k).collect(); + cleanup(provider, &ns, &keys).await; +} + +/// A namespace unique to this driver and assertion. +/// +/// Prefixed so the suite can run against a live service holding real data +/// without colliding with it, and without needing a teardown it might not get. +fn ns(provider: &dyn MemoryProvider, what: &str) -> String { + format!("tinymemory-conformance/{}/{what}", provider.driver_id()) +} + +/// Best-effort teardown. Failures are ignored: a driver that cannot delete is +/// reported by [`assert_forget_is_idempotent`], and failing here would mask the +/// assertion that actually found the problem. +async fn cleanup(provider: &dyn MemoryProvider, namespace: &str, keys: &[&str]) { + for key in keys { + let _ = provider.forget(namespace, key).await; + } +} diff --git a/conformance/tests/reference_drivers.rs b/conformance/tests/reference_drivers.rs new file mode 100644 index 0000000..fe8be6d --- /dev/null +++ b/conformance/tests/reference_drivers.rs @@ -0,0 +1,47 @@ +//! The suite, run against the drivers this workspace ships as references. +//! +//! Two drivers, for two different reasons. +//! +//! `InMemoryProvider` is the calibration subject: its behaviour is obvious by +//! inspection, so a failure here means the *assertion* is wrong, not the +//! driver. Without it, a suite that only ever ran against real engines could +//! not tell those two cases apart. +//! +//! `NullMemoryProvider` is the opposite end — it accepts writes, discards them, +//! and reads back empty. Running the same assertions against it pins down which +//! parts of the contract a discard-everything driver must still uphold +//! (namespace isolation, an honest `forget`, a terminating export cursor, +//! errors that stay inside `MemoryError`) and which are vacuous for it. +//! A suite that could not run against `null` would be asserting storage rather +//! than the contract. + +use std::sync::Arc; + +use tinymemory_api::null::NullMemoryProvider; +use tinymemory_api::provider::MemoryProvider; +use tinymemory_conformance::{assert_provider, InMemoryProvider}; + +#[tokio::test] +async fn the_in_memory_reference_driver_conforms() { + assert_provider(Arc::new(InMemoryProvider::new())).await; +} + +#[tokio::test] +async fn the_null_driver_conforms() { + assert_provider(Arc::new(NullMemoryProvider::new())).await; +} + +#[tokio::test] +async fn the_reference_driver_advertises_exactly_the_mandatory_families() { + let provider = InMemoryProvider::new(); + let caps = provider.capabilities(); + assert_eq!( + caps.len(), + 3, + "the reference driver must advertise only what it can serve, got {caps:?}" + ); + // Every optional accessor stays `None`, which is what makes the audit pass. + assert!(provider.as_tree().is_none()); + assert!(provider.as_graph().is_none()); + assert!(provider.as_ingest().is_none()); +} diff --git a/core/Cargo.toml b/core/Cargo.toml index 18fd2f7..628cafd 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -6,7 +6,7 @@ name = "tinymemory-core" publish = false version = "0.1.0" edition = "2021" -rust-version = "1.85" +rust-version = "1.96" license = "MIT" description = "The engine-neutral memory subsystem: store, summary tree, sync pipelines, ingestion and recall" repository = "https://github.com/tinyhumansai/tinymemory" diff --git a/core/src/store/namespace_store/events.rs b/core/src/store/namespace_store/events.rs index bb3aca4..c03bf07 100644 --- a/core/src/store/namespace_store/events.rs +++ b/core/src/store/namespace_store/events.rs @@ -475,7 +475,7 @@ fn decode_embedding_row(bytes: &[u8], dim: i64) -> anyhow::Result anyhow::Result Vec { /// [`EMBEDDING_DIM`] (after decoding). The latter guards against rows /// written with a mismatched-provider blob silently passing as valid. pub fn unpack_embedding(b: &[u8]) -> Result> { - if b.len() % 4 != 0 { + if !b.len().is_multiple_of(4) { anyhow::bail!( "embedding blob length {} not a multiple of 4 — corrupt row", b.len() diff --git a/crates/tinymemory-module/Cargo.lock b/crates/tinymemory-module/Cargo.lock index 30ffb20..e5599eb 100644 --- a/crates/tinymemory-module/Cargo.lock +++ b/crates/tinymemory-module/Cargo.lock @@ -2027,9 +2027,16 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "chrono", + "log", + "serde", + "serde_json", "tinycortex", "tinymemory", "tinymemory-api", + "tinymemory-core", + "tokio", + "uuid", ] [[package]] diff --git a/crates/tinymemory-module/Cargo.toml b/crates/tinymemory-module/Cargo.toml index c8482f3..adbc845 100644 --- a/crates/tinymemory-module/Cargo.toml +++ b/crates/tinymemory-module/Cargo.toml @@ -32,7 +32,7 @@ tinymemory = { path = "../.." } # the module: they are 14.7s of the host's critical build path, and a host that # loads this binary compiles neither. tinymemory-core = { path = "../../core", features = ["memory-git"] } -tinymemory-tinycortex = { path = "../../adapters/tinycortex" } +tinymemory-tinycortex = { path = "../../adapters/tinycortex", features = ["memory-git"] } # `people` is enabled here rather than inherited: the module serves the # `MemoryPeople` family directly off the engine's people store, so it needs the # gate on even though `tinymemory-core` only re-exports the domain. diff --git a/crates/tinymemory-module/src/lib.rs b/crates/tinymemory-module/src/lib.rs index 9cad0eb..30e705f 100644 --- a/crates/tinymemory-module/src/lib.rs +++ b/crates/tinymemory-module/src/lib.rs @@ -153,7 +153,7 @@ async fn setup(connection: Connection, mut config: ModuleConfig) -> BusResult<() setup_error("create memory store") })?; - let provider = provider::ModuleMemoryProvider::new(&config, Arc::new(client)); + let provider = provider::provider(&config, Arc::new(client)); service::serve(&connection, Arc::new(provider), config).await } diff --git a/crates/tinymemory-module/src/provider.rs b/crates/tinymemory-module/src/provider.rs index 48e46a2..278a000 100644 --- a/crates/tinymemory-module/src/provider.rs +++ b/crates/tinymemory-module/src/provider.rs @@ -1,66 +1,18 @@ -//! Complete TinyMemory provider backed by the module-owned engine. +//! The module's own configuration, converted for the engine provider. +//! +//! The provider itself now lives in `tinymemory-tinycortex` (issue #18 §C3). +//! Everything that was here delegated to `tinymemory-core` on a blocking +//! thread and was never module-specific; what remains is the one thing that is +//! — turning a `ModuleConfig` into the engine's runtime configuration. -use std::collections::HashSet; -use std::path::PathBuf; use std::sync::Arc; -use async_trait::async_trait; -use chrono::Utc; -use tinymemory::mandatory::MemoryTraitProvider; -use tinymemory_api::capabilities::Capabilities; -use tinymemory_api::chunks::Chunk; -use tinymemory_api::error::MemoryError; -use tinymemory_api::goals::GoalsDoc; -use tinymemory_api::health::MemoryHealth; -use tinymemory_api::host::{ - CloudProviderCreds, ComposioMode, LocalAiConfig, MemoryConfig, MemoryHostConfig, - MemoryTreeConfig, SchedulerGateConfig, -}; -use tinymemory_api::provider::types::{ - ChangeKind, DiffReport, EntityHit, EntityRef, ExportPage, ExportRecord, ImportOutcome, - IngestItem, IngestOutcome, MaintenanceReport, SnapshotRef, SourceChange, SourceItem, - SourceScope, -}; -use tinymemory_api::provider::{ - AddressBookSeedOutcome, ChunkDetail, ChunkEmbedding, ChunkQuery, ConversationSegment, - CoverWindowQuery, EntityMatch, EpisodicTurn, FacetType, FastRetrieveQuery, MemoryChunks, - MemoryCore, MemoryDiff, MemoryDocuments, MemoryEntities, MemoryEpisodic, MemoryGoals, - MemoryGraph, MemoryIngest, MemoryMaintenance, MemoryPeople, MemoryPortability, MemoryProfile, - MemoryProvider, MemoryRecall, MemoryRetrieval, MemorySourceSink, MemoryToolMemory, MemoryTree, - PersonHandle, PersonInteraction, PersonRecord, PersonScore, ProfileFacet, RankedPerson, - ResolvedPerson, RetrievalHit, RetrievalResponse, SourceRetrievalQuery, UserState, -}; -use tinymemory_api::recall::OwnedRecallOpts; -use tinymemory_api::tool_memory::ToolMemoryRule; -use tinymemory_api::tree::{IngestRequest, QueryResult, TreeStatus}; -use tinymemory_api::types::{ - GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, - NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext, NamespaceSummary, - StoredMemoryDocument, -}; -use tinymemory_core::store::{MemoryClient, MemoryClientRef}; -use tinymemory_tinycortex::TinycortexMemory; +use tinymemory_core::store::MemoryClient; +use tinymemory_tinycortex::engine::{EngineRuntimeConfig, TinycortexProvider}; use crate::ModuleConfig; -/// The concrete, credential-free host configuration available inside a module. -#[derive(Debug, Clone)] -struct ModuleRuntimeConfig { - workspace_dir: PathBuf, - config_path: PathBuf, - memory: MemoryConfig, - memory_tree: MemoryTreeConfig, - scheduler_gate: SchedulerGateConfig, - local_ai: LocalAiConfig, - embeddings_provider: Option, - memory_provider: Option, - default_model: Option, - default_temperature: f64, - output_language: Option, - memory_sources: serde_json::Value, -} - -impl From<&ModuleConfig> for ModuleRuntimeConfig { +impl From<&ModuleConfig> for EngineRuntimeConfig { fn from(config: &ModuleConfig) -> Self { Self { workspace_dir: config.workspace_dir.clone(), @@ -79,2111 +31,11 @@ impl From<&ModuleConfig> for ModuleRuntimeConfig { } } -#[async_trait] -impl MemoryHostConfig for ModuleRuntimeConfig { - fn workspace_dir(&self) -> &PathBuf { - &self.workspace_dir - } - fn config_path(&self) -> &PathBuf { - &self.config_path - } - fn memory_tree_content_root(&self) -> PathBuf { - self.memory_tree - .content_dir - .clone() - .unwrap_or_else(|| self.workspace_dir.join("memory_tree/content")) - } - fn memory(&self) -> &MemoryConfig { - &self.memory - } - fn memory_tree(&self) -> &MemoryTreeConfig { - &self.memory_tree - } - fn scheduler_gate(&self) -> &SchedulerGateConfig { - &self.scheduler_gate - } - fn local_ai(&self) -> &LocalAiConfig { - &self.local_ai - } - fn cloud_providers(&self) -> &Vec { - static NONE: Vec = Vec::new(); - &NONE - } - fn embeddings_provider(&self) -> Option<&str> { - self.embeddings_provider.as_deref() - } - fn memory_provider(&self) -> Option<&str> { - self.memory_provider.as_deref() - } - fn workload_local_model(&self, workload: &str) -> Option { - let route = match workload { - "memory" => self.memory_provider.as_deref(), - "embeddings" => self.embeddings_provider.as_deref(), - _ => None, - }?; - route - .strip_prefix("ollama:") - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) - } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn to_arc(&self) -> Arc { - Arc::new(self.clone()) - } - fn api_url(&self) -> Option<&str> { - None - } - fn effective_backend_api_url(&self) -> String { - String::new() - } - fn session_token(&self) -> Result, String> { - Ok(None) - } - fn default_model(&self) -> Option<&str> { - self.default_model.as_deref() - } - fn default_temperature(&self) -> f64 { - self.default_temperature - } - fn output_language(&self) -> Option<&str> { - self.output_language.as_deref() - } - fn memory_sync_interval_secs(&self) -> Option { - Some(0) - } - fn onboarding_completed(&self) -> bool { - true - } - fn secrets_encrypt(&self) -> bool { - false - } - fn composio(&self) -> ComposioMode { - ComposioMode::default() - } - fn memory_sources_json(&self) -> anyhow::Result { - Ok(self.memory_sources.clone()) - } - fn set_memory_sources_json(&mut self, value: serde_json::Value) -> anyhow::Result<()> { - self.memory_sources = value; - Ok(()) - } - fn composio_source_caps_migration_version(&self) -> u32 { - 0 - } - fn set_composio_source_caps_migration_version(&mut self, _version: u32) {} - fn apply_env_overrides(&mut self) {} - async fn save(&self) -> anyhow::Result<()> { - Ok(()) - } -} - -/// The module-owned implementation of every TinyMemory capability family. -pub(crate) struct ModuleMemoryProvider { - driver_id: String, - mandatory: MemoryTraitProvider, - client: MemoryClientRef, - config: ModuleRuntimeConfig, -} - -impl ModuleMemoryProvider { - pub(crate) fn new(config: &ModuleConfig, client: Arc) -> Self { - let memory = client.memory_handle(); - let mandatory = MemoryTraitProvider::new( - Arc::new(TinycortexMemory::new(memory)), - config.driver_id.clone(), - ); - Self { - driver_id: config.driver_id.clone(), - mandatory, - client, - config: ModuleRuntimeConfig::from(config), - } - } - - fn other(context: &'static str, error: impl std::fmt::Display) -> MemoryError { - MemoryError::Other(anyhow::anyhow!("{context}: {error}")) - } - - fn cross( - value: &A, - context: &'static str, - ) -> Result { - let value = serde_json::to_value(value).map_err(|error| Self::other(context, error))?; - serde_json::from_value(value).map_err(|error| Self::other(context, error)) - } -} - -fn validate_ingest_item(item: &IngestItem) -> Result<(), MemoryError> { - if item.taint != MemoryTaint::default() { - return Err(MemoryError::Invalid( - "ingest cannot preserve a non-default taint in the chunk tier".to_string(), - )); - } - if item.content.trim().is_empty() { - return Err(MemoryError::Invalid( - "ingest content must not be empty".to_string(), - )); - } - if let Some(mime) = item.mime.as_deref() { - let mime = mime.trim().to_ascii_lowercase(); - let base = mime.split(';').next().unwrap_or("").trim(); - if !(base.starts_with("text/") - || base.ends_with("+json") - || base.ends_with("+xml") - || matches!( - base, - "application/json" | "application/xml" | "application/x-ndjson" - )) - { - return Err(MemoryError::Invalid(format!( - "unsupported MIME '{mime}': ingest accepts decoded text only" - ))); - } - } - Ok(()) -} - -async fn blocking( - config: ModuleRuntimeConfig, - context: &'static str, - run: F, -) -> Result -where - T: Send + 'static, - F: FnOnce(&ModuleRuntimeConfig) -> anyhow::Result + Send + 'static, -{ - tokio::task::spawn_blocking(move || run(&config)) - .await - .map_err(|error| ModuleMemoryProvider::other(context, error))? - .map_err(|error| ModuleMemoryProvider::other(context, error)) -} - -#[async_trait] -impl MemoryCore for ModuleMemoryProvider { - async fn store( - &self, - namespace: &str, - key: &str, - content: &str, - category: MemoryCategory, - session_id: Option<&str>, - taint: MemoryTaint, - ) -> Result<(), MemoryError> { - self.mandatory - .store(namespace, key, content, category, session_id, taint) - .await - } - async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { - self.mandatory.get(namespace, key).await - } - async fn forget(&self, namespace: &str, key: &str) -> Result { - self.mandatory.forget(namespace, key).await - } - async fn list( - &self, - namespace: Option<&str>, - category: Option<&MemoryCategory>, - session_id: Option<&str>, - ) -> Result, MemoryError> { - self.mandatory.list(namespace, category, session_id).await - } - async fn namespaces(&self) -> Result, MemoryError> { - self.mandatory.namespaces().await - } -} - -#[async_trait] -impl MemoryRecall for ModuleMemoryProvider { - async fn recall( - &self, - query: &str, - limit: usize, - opts: &OwnedRecallOpts, - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - self.mandatory.recall(query, limit, opts, scope).await - } -} - -#[async_trait] -impl MemoryPortability for ModuleMemoryProvider { - async fn export_page( - &self, - cursor: Option<&str>, - limit: usize, - ) -> Result { - self.mandatory.export_page(cursor, limit).await - } - async fn import_records( - &self, - records: Vec, - ) -> Result { - self.mandatory.import_records(records).await - } -} - -#[async_trait] -impl MemoryDocuments for ModuleMemoryProvider { - async fn put_document(&self, input: NamespaceDocumentInput) -> Result { - let input = Self::cross(&input, "convert document input")?; - self.client - .put_doc(input) - .await - .map_err(|error| Self::other("put_document", error)) - } - async fn get_document( - &self, - namespace: &str, - key: &str, - ) -> Result, MemoryError> { - let document = self - .client - .get_document(namespace, key) - .await - .map_err(|error| Self::other("get_document", error))?; - document - .map(|document| Self::cross(&document, "convert stored document")) - .transpose() - } - - async fn list_documents( - &self, - namespace: Option<&str>, - ) -> Result { - self.client - .list_documents(namespace) - .await - .map_err(|error| Self::other("list_documents", error)) - } - - async fn list_namespaces(&self) -> Result, MemoryError> { - self.client - .list_namespaces() - .await - .map_err(|error| Self::other("list_namespaces", error)) - } - - async fn delete_document( - &self, - namespace: &str, - document_id: &str, - ) -> Result { - self.client - .delete_document(namespace, document_id) - .await - .map_err(|error| Self::other("delete_document", error)) - } - - async fn clear_namespace(&self, namespace: &str) -> Result<(), MemoryError> { - self.client - .clear_namespace(namespace) - .await - .map_err(|error| Self::other("clear_namespace", error)) - } - async fn query_documents( - &self, - namespace: &str, - query: &str, - limit: usize, - ) -> Result { - let limit = u32::try_from(limit).unwrap_or(u32::MAX); - let context = self - .client - .query_namespace_context_data(namespace, query, limit) - .await - .map_err(|error| Self::other("query_documents", error))?; - Self::cross(&context, "convert document query result") - } - - async fn recall_documents( - &self, - namespace: &str, - limit: usize, - ) -> Result { - let limit = u32::try_from(limit).unwrap_or(u32::MAX); - let context = self - .client - .recall_namespace_context_data(namespace, limit) - .await - .map_err(|error| Self::other("recall_documents", error))?; - Self::cross(&context, "convert document recall result") - } -} - -#[async_trait] -impl MemoryIngest for ModuleMemoryProvider { - async fn ingest_document(&self, item: IngestItem) -> Result { - validate_ingest_item(&item)?; - let document = tinycortex::memory::ingest::canonicalize::document::DocumentInput { - provider: item.source.as_str().to_string(), - title: String::new(), - body: item.content, - modified_at: item.timestamp.unwrap_or_else(Utc::now), - source_ref: item.source_ref.map(|source_ref| source_ref.value), - }; - let result = tinymemory_core::ingest_pipeline::ingest_document_with_scope( - &self.config, - &item.source_id, - &item.owner, - item.tags, - document, - item.path_scope, - ) - .await - .map_err(|error| Self::other("ingest document", error))?; - Ok(IngestOutcome { - written: u32::try_from(result.chunks_written).unwrap_or(u32::MAX), - skipped: if result.already_ingested { - 1 - } else { - u32::try_from(result.chunks_dropped).unwrap_or(u32::MAX) - }, - ids: result.chunk_ids, - }) - } - - async fn ingest_chat(&self, messages: Vec) -> Result { - let Some(first) = messages.first() else { - return Ok(IngestOutcome::default()); - }; - let source_id = first.source_id.clone(); - let owner = first.owner.clone(); - let tags = first.tags.clone(); - let platform = first.source.as_str().to_string(); - for item in &messages { - validate_ingest_item(item)?; - if item.source_id != source_id { - return Err(MemoryError::Invalid( - "ingest_chat batches must contain one conversation".to_string(), - )); - } - } - let batch = tinycortex::memory::ingest::canonicalize::chat::ChatBatch { - platform, - channel_label: source_id.clone(), - messages: messages - .into_iter() - .map( - |item| tinycortex::memory::ingest::canonicalize::chat::ChatMessage { - author: item.owner, - timestamp: item.timestamp.unwrap_or_else(Utc::now), - text: item.content, - source_ref: item.source_ref.map(|source_ref| source_ref.value), - }, - ) - .collect(), - }; - let result = tinymemory_core::ingest_pipeline::ingest_chat( - &self.config, - &source_id, - &owner, - tags, - batch, - ) - .await - .map_err(|error| Self::other("ingest chat", error))?; - Ok(IngestOutcome { - written: u32::try_from(result.chunks_written).unwrap_or(u32::MAX), - skipped: if result.already_ingested { - 1 - } else { - u32::try_from(result.chunks_dropped).unwrap_or(u32::MAX) - }, - ids: result.chunk_ids, - }) - } -} - -#[async_trait] -impl MemoryGraph for ModuleMemoryProvider { - async fn kv_get( - &self, - namespace: Option<&str>, - key: &str, - ) -> Result, MemoryError> { - let record = self - .client - .kv_records(namespace) - .await - .map_err(|error| Self::other("kv_get", error))? - .into_iter() - .find(|record| record.key == key); - record - .map(|record| Self::cross(&record, "convert key/value record")) - .transpose() - } - async fn kv_put( - &self, - namespace: Option<&str>, - key: &str, - value: serde_json::Value, - ) -> Result<(), MemoryError> { - self.client - .kv_set(namespace, key, &value) - .await - .map_err(|error| Self::other("kv_put", error)) - } - - async fn kv_delete(&self, namespace: Option<&str>, key: &str) -> Result { - self.client - .kv_delete(namespace, key) - .await - .map_err(|error| Self::other("kv_delete", error)) - } - async fn kv_list( - &self, - namespace: Option<&str>, - prefix: Option<&str>, - limit: usize, - ) -> Result, MemoryError> { - let mut records = self - .client - .kv_records(namespace) - .await - .map_err(|error| Self::other("kv_list", error))?; - if let Some(prefix) = prefix { - records.retain(|record| record.key.starts_with(prefix)); - } - records.truncate(limit); - Self::cross(&records, "convert key/value records") - } - async fn relations( - &self, - namespace: Option<&str>, - subject: Option<&str>, - predicate: Option<&str>, - limit: usize, - ) -> Result, MemoryError> { - let mut records = self - .client - .graph_relations(namespace, subject, predicate) - .await - .map_err(|error| Self::other("relations", error))?; - records.truncate(limit); - Self::cross(&records, "convert graph relations") - } - async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError> { - self.client - .graph_upsert( - relation.namespace.as_deref(), - &relation.subject, - &relation.predicate, - &relation.object, - &relation.attrs, - ) - .await - .map_err(|error| Self::other("put_relation", error)) - } -} - -#[async_trait] -impl MemoryGoals for ModuleMemoryProvider { - async fn goals(&self) -> Result { - let workspace = self.config.workspace_dir.clone(); - let document = - tokio::task::spawn_blocking(move || tinycortex::memory::goals::store::load(&workspace)) - .await - .map_err(|error| Self::other("join goals read", error))? - .map_err(|error| Self::other("read goals", error))?; - Self::cross(&document, "convert goals") - } - - async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError> { - let workspace = self.config.workspace_dir.clone(); - let mut goals = Self::cross(&goals, "convert goals")?; - tokio::task::spawn_blocking(move || { - tinycortex::memory::goals::store::save(&workspace, &mut goals) - }) - .await - .map_err(|error| Self::other("join goals write", error))? - .map_err(|error| Self::other("write goals", error)) - } -} - -#[async_trait] -impl MemoryToolMemory for ModuleMemoryProvider { - async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError> { - let rules = tinymemory_core::tool_memory::tool_memory_store(self.client.memory_handle()) - .list_rules(tool_name) - .await - .map_err(|error| Self::other("list tool rules", error))?; - Self::cross(&rules, "convert tool rules") - } - - async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError> { - let rule = Self::cross(&rule, "convert tool rule")?; - tinymemory_core::tool_memory::tool_memory_store(self.client.memory_handle()) - .put_rule(rule) - .await - .map(|_| ()) - .map_err(|error| Self::other("put tool rule", error)) - } - - async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result { - tinymemory_core::tool_memory::tool_memory_store(self.client.memory_handle()) - .delete_rule(tool_name, rule_id) - .await - .map_err(|error| Self::other("delete tool rule", error)) - } -} - -#[async_trait] -impl MemoryTree for ModuleMemoryProvider { - async fn append(&self, request: IngestRequest) -> Result<(), MemoryError> { - tinycortex::memory::tree::runtime::store::validate_namespace(&request.namespace) - .map_err(MemoryError::Invalid)?; - if request.content.trim().is_empty() { - return Err(MemoryError::Invalid( - "content must not be empty".to_string(), - )); - } - let namespace = request.namespace.trim().to_string(); - let content = request.content; - let timestamp = request.timestamp.unwrap_or_else(Utc::now); - let metadata = request.metadata; - blocking(self.config.clone(), "append tree content", move |config| { - tinymemory_core::tree::tree_runtime::store::buffer_write( - config, - &namespace, - &content, - ×tamp, - metadata.as_ref(), - ) - .map(|_| ()) - }) - .await - } - - async fn query_source( - &self, - namespace: &str, - source_id: &str, - limit: usize, - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - tinycortex::memory::tree::runtime::store::validate_namespace(namespace) - .map_err(MemoryError::Invalid)?; - let query = tinymemory_core::store::chunks::ListChunksQuery { - source_id: Some(source_id.to_string()), - source_scope: scope.map(|scope| scope.allow.iter().cloned().collect::>()), - limit: Some(limit), - exclude_dropped: true, - ..Default::default() - }; - let chunks = blocking(self.config.clone(), "query source", move |config| { - tinymemory_core::store::chunks::list_chunks(config, &query) - }) - .await?; - Self::cross(&chunks, "convert source chunks") - } - - async fn drill_down(&self, namespace: &str, node_id: &str) -> Result { - tinycortex::memory::tree::runtime::store::validate_namespace(namespace) - .map_err(MemoryError::Invalid)?; - tinycortex::memory::tree::runtime::store::validate_node_id(node_id) - .map_err(MemoryError::Invalid)?; - let namespace = namespace.trim().to_string(); - let node_id = node_id.to_string(); - let lookup_namespace = namespace.clone(); - let lookup_node = node_id.clone(); - let result = blocking(self.config.clone(), "drill down", move |config| { - let Some(node) = tinymemory_core::tree::tree_runtime::store::read_node( - config, - &lookup_namespace, - &lookup_node, - )? - else { - return Ok(None); - }; - let children = tinymemory_core::tree::tree_runtime::store::read_children( - config, - &lookup_namespace, - &lookup_node, - )?; - Ok(Some((node, children))) - }) - .await? - .ok_or_else(|| { - MemoryError::NotFound(format!("tree node '{node_id}' not found in '{namespace}'")) - })?; - Self::cross(&result, "convert tree drill-down") - .map(|(node, children)| QueryResult { node, children }) - } - - async fn seal(&self, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::validate_namespace(namespace) - .map_err(MemoryError::Invalid)?; - let namespace = namespace.trim().to_string(); - let read_namespace = namespace.clone(); - let buffered = blocking(self.config.clone(), "read tree buffer", move |config| { - tinymemory_core::tree::tree_runtime::store::buffer_read(config, &read_namespace) - }) - .await?; - if !buffered.is_empty() { - let (model, _) = tinymemory_core::chat_host::create_chat_model_with_model_id( - "summarization", - &self.config, - self.config.default_temperature, - ) - .map_err(|error| Self::other("create summarizer", error))?; - tinymemory_core::tree::tree_runtime::engine::run_summarization( - &self.config, - model.as_ref(), - &namespace, - Utc::now(), - ) - .await - .map_err(|error| Self::other("seal tree", error))?; - } - let status = blocking(self.config.clone(), "read tree status", move |config| { - tinymemory_core::tree::tree_runtime::store::get_tree_status(config, &namespace) - }) - .await?; - Self::cross(&status, "convert tree status") - } - - async fn cascade(&self, namespace: &str) -> Result { - tinycortex::memory::tree::runtime::store::validate_namespace(namespace) - .map_err(MemoryError::Invalid)?; - let namespace = namespace.trim().to_string(); - let read_namespace = namespace.clone(); - let status = blocking(self.config.clone(), "read tree status", move |config| { - tinymemory_core::tree::tree_runtime::store::get_tree_status(config, &read_namespace) - }) - .await?; - if status.total_nodes == 0 { - return Self::cross(&status, "convert tree status"); - } - let (model, _) = tinymemory_core::chat_host::create_chat_model_with_model_id( - "summarization", - &self.config, - self.config.default_temperature, - ) - .map_err(|error| Self::other("create summarizer", error))?; - let status = tinymemory_core::tree::tree_runtime::engine::rebuild_tree( - &self.config, - model.as_ref(), - &namespace, - ) - .await - .map_err(|error| Self::other("cascade tree", error))?; - Self::cross(&status, "convert tree status") - } -} - -#[async_trait] -impl MemoryEntities for ModuleMemoryProvider { - async fn entities( - &self, - namespace: &str, - query: Option<&str>, - limit: usize, - ) -> Result, MemoryError> { - let namespace = namespace.to_string(); - let query_namespace = namespace.clone(); - let query = query.map(str::to_string); - let rows = blocking( - self.config.clone(), - "list namespace entities", - move |config| { - tinymemory_core::store::entities::namespace_entities( - config, - &query_namespace, - query.as_deref(), - limit, - ) - }, - ) - .await? - .into_iter() - .map(|hit| (hit.id, hit.kind, hit.name, hit.mentions)) - .collect::>(); - - let config = self.config.clone(); - blocking(config, "attach entity hotness", move |config| { - Ok(rows - .into_iter() - .map(|(id, kind, name, mentions)| { - let hotness_key = format!("{namespace}:{id}"); - let hotness = tinymemory_core::store::trees::hotness::get(config, &hotness_key) - .ok() - .flatten() - .map_or(0.0, |counters| { - f64::from( - tinymemory_core::tree_policy::TreePolicy::topic().topic_hotness( - &id, - &counters.stats(), - Utc::now().timestamp_millis(), - ), - ) - }); - EntityHit { - entity: EntityRef { id, kind, name }, - hotness, - mentions, - } - }) - .collect()) - }) - .await - } - - async fn entity_edges( - &self, - namespace: &str, - entity_id: &str, - limit: usize, - ) -> Result, MemoryError> { - let subject = entity_id.to_string(); - let lookup = subject.clone(); - let namespace = namespace.to_string(); - let query_namespace = namespace.clone(); - let neighbours = blocking(self.config.clone(), "read entity edges", move |config| { - tinymemory_core::store::entities::namespace_entity_edges( - config, - &query_namespace, - &lookup, - limit, - ) - }) - .await?; - Ok(neighbours - .into_iter() - .map(|(object, weight)| GraphRelationRecord { - namespace: Some(namespace.clone()), - subject: subject.clone(), - predicate: "co_occurs_with".to_string(), - object, - attrs: serde_json::Value::Null, - updated_at: 0.0, - evidence_count: weight, - order_index: None, - document_ids: Vec::new(), - chunk_ids: Vec::new(), - }) - .collect()) - } - - async fn touch_entities( - &self, - namespace: &str, - entity_ids: &[String], - ) -> Result<(), MemoryError> { - let entity_ids = entity_ids.to_vec(); - let namespace = namespace.to_string(); - blocking(self.config.clone(), "touch entities", move |config| { - let now = Utc::now().timestamp_millis(); - for entity_id in entity_ids { - let entity_id = format!("{namespace}:{entity_id}"); - let mut counters = - tinymemory_core::store::trees::hotness::get_or_fresh(config, &entity_id)?; - counters.mention_count_30d = counters.mention_count_30d.saturating_add(1); - counters.last_seen_ms = Some(now); - counters.last_updated_ms = now; - tinymemory_core::store::trees::hotness::upsert(config, &counters)?; - } - Ok(()) - }) - .await - } -} - -#[async_trait] -impl MemoryDiff for ModuleMemoryProvider { - async fn capture_snapshot(&self, source_id: &str) -> Result { - let source = tinymemory_core::sources::registry::decode_memory_sources(&self.config) - .into_iter() - .find(|source| source.id == source_id) - .ok_or_else(|| MemoryError::NotFound(source_id.to_string()))?; - let snapshot = tinymemory_core::diff::ops::take_snapshot( - &source, - &self.config, - tinymemory_core::diff::SnapshotTrigger::Manual, - ) - .await - .map_err(|error| Self::other("capture snapshot", error))?; - Ok(SnapshotRef { - id: snapshot.id, - source_id: snapshot.source_id, - label: snapshot.label, - item_count: snapshot.item_count, - taken_at_ms: snapshot.taken_at_ms, - }) - } - - async fn snapshots( - &self, - source_id: &str, - limit: usize, - ) -> Result, MemoryError> { - let snapshots = tinymemory_core::diff::ops::list_snapshots( - &self.config, - Some(source_id), - u32::try_from(limit).unwrap_or(u32::MAX), - ) - .await - .map_err(|error| Self::other("list snapshots", error))?; - Ok(snapshots - .into_iter() - .map(|snapshot| SnapshotRef { - id: snapshot.id, - source_id: snapshot.source_id, - label: snapshot.label, - item_count: snapshot.item_count, - taken_at_ms: snapshot.taken_at_ms, - }) - .collect()) - } - - async fn diff( - &self, - source_id: &str, - from: Option<&str>, - to: &str, - ) -> Result { - let result = tinymemory_core::diff::ops::compute_diff(&self.config, from, to, false) - .await - .map_err(|error| Self::other("compute diff", error))?; - if result.source_id != source_id { - return Err(MemoryError::Invalid(format!( - "snapshot '{to}' belongs to a different source" - ))); - } - let changes = result - .changes - .into_iter() - .map(|change| SourceChange { - item_id: change.item_id, - title: change.title, - kind: match change.kind { - tinymemory_core::diff::ChangeKind::Added => ChangeKind::Added, - tinymemory_core::diff::ChangeKind::Removed => ChangeKind::Removed, - tinymemory_core::diff::ChangeKind::Modified => ChangeKind::Modified, - }, - old_content_hash: change.old_content_hash, - new_content_hash: change.new_content_hash, - }) - .collect(); - Ok(DiffReport { - source_id: result.source_id, - from_snapshot_id: result.from_snapshot_id, - to_snapshot_id: result.to_snapshot_id, - added: result.summary.added, - removed: result.summary.removed, - modified: result.summary.modified, - unchanged: result.summary.unchanged, - changes, - }) - } -} - -#[async_trait] -impl MemorySourceSink for ModuleMemoryProvider { - async fn accept_source_items( - &self, - source_id: &str, - source_kind: &str, - items: Vec, - taint: MemoryTaint, - ) -> Result { - let namespace = format!("source:{source_id}"); - let mut outcome = IngestOutcome::default(); - for item in items { - if item.item_id.trim().is_empty() { - return Err(MemoryError::Invalid( - "source item_id must not be empty".to_string(), - )); - } - let title = if item.title.trim().is_empty() { - item.item_id.clone() - } else { - item.title.clone() - }; - let input = NamespaceDocumentInput { - namespace: namespace.clone(), - key: item.item_id, - title, - content: item.content, - source_type: source_kind.to_string(), - priority: "medium".to_string(), - tags: item.tags, - metadata: serde_json::json!({ - "sourceId": source_id, - "sourceKind": source_kind, - "url": item.url, - "mime": item.mime, - "updatedAtMs": item.updated_at_ms, - }), - category: "core".to_string(), - session_id: None, - document_id: None, - taint, - }; - let input = Self::cross(&input, "convert source document")?; - match self.client.put_doc(input).await { - Ok(id) => { - outcome.written = outcome.written.saturating_add(1); - outcome.ids.push(id); - } - Err(_) => { - outcome.skipped = outcome.skipped.saturating_add(1); - } - } - } - Ok(outcome) - } - - async fn forget_source(&self, source_id: &str) -> Result { - let namespace = format!("source:{source_id}"); - let listed = self - .client - .list_documents(Some(&namespace)) - .await - .map_err(|error| Self::other("list source documents", error))?; - let documents = listed - .get("documents") - .and_then(serde_json::Value::as_array) - .map_or(0, Vec::len); - if documents > 0 { - self.client - .clear_namespace(&namespace) - .await - .map_err(|error| Self::other("clear source documents", error))?; - } - let source_id = source_id.to_string(); - let chunks = blocking(self.config.clone(), "clear source chunks", move |config| { - use tinymemory_core::store::chunks::{ - delete_chunks_by_source, delete_orphaned_source_tree, SourceKind, - }; - let removed = delete_chunks_by_source(config, SourceKind::Document, &source_id)?; - delete_orphaned_source_tree(config, SourceKind::Document, &source_id)?; - Ok(removed) - }) - .await?; - Ok(u64::try_from(documents.saturating_add(chunks)).unwrap_or(u64::MAX)) - } -} - -#[async_trait] -impl MemoryMaintenance for ModuleMemoryProvider { - async fn reembed(&self) -> Result { - let (examined, changed) = - blocking(self.config.clone(), "enqueue re-embedding", move |config| { - let total = tinymemory_core::queue::count_total(config).unwrap_or(0); - let before = tinymemory_core::queue::count_by_status( - config, - tinymemory_core::queue::JobStatus::Ready, - ) - .unwrap_or(0); - tinymemory_core::queue::ensure_reembed_backfill(config); - let after = tinymemory_core::queue::count_by_status( - config, - tinymemory_core::queue::JobStatus::Ready, - ) - .unwrap_or(0); - Ok((total, after.saturating_sub(before))) - }) - .await?; - Ok(MaintenanceReport { - operation: "reembed".to_string(), - examined, - changed, - findings: vec![format!("enqueued {changed} re-embedding job(s)")], - }) - } - - async fn compact(&self) -> Result { - let (examined, changed) = - blocking(self.config.clone(), "compact memory queue", move |config| { - Ok(( - tinymemory_core::queue::count_total(config).unwrap_or(0), - u64::try_from(tinymemory_core::queue::recover_stale_locks(config).unwrap_or(0)) - .unwrap_or(u64::MAX), - )) - }) - .await?; - Ok(MaintenanceReport { - operation: "compact".to_string(), - examined, - changed, - findings: vec![format!("released {changed} stale queue lock(s)")], - }) - } - - async fn consolidate(&self) -> Result { - let (examined, enqueued) = blocking( - self.config.clone(), - "enqueue consolidation", - move |config| { - Ok(( - tinymemory_core::queue::count_total(config).unwrap_or(0), - tinymemory_core::queue::scheduler::enqueue_flush_stale_job(config) - .map_err(anyhow::Error::msg)?, - )) - }, - ) - .await?; - Ok(MaintenanceReport { - operation: "consolidate".to_string(), - examined, - changed: u64::from(enqueued), - findings: vec![if enqueued { - "enqueued a stale-buffer flush".to_string() - } else { - "a stale-buffer flush is already queued".to_string() - }], - }) - } - - async fn doctor(&self) -> Result { - let report = tinymemory_core::tree::health::async_run_doctor(&self.config).await; - Ok(MaintenanceReport { - operation: "doctor".to_string(), - examined: report.counters.total_chunks, - changed: 0, - findings: report - .stages - .into_iter() - .filter(|stage| !stage.ok) - .map(|stage| format!("{}: {}", stage.stage, stage.note)) - .collect(), - }) - } -} - -#[async_trait] -impl MemoryProvider for ModuleMemoryProvider { - fn driver_id(&self) -> &str { - &self.driver_id - } - fn capabilities(&self) -> Capabilities { - Capabilities::all() - } - async fn health(&self) -> MemoryHealth { - if self.client.memory_handle().health_check().await { - MemoryHealth::Ready - } else { - MemoryHealth::down("memory store is unavailable") - } - } - fn as_documents(&self) -> Option<&dyn MemoryDocuments> { - Some(self) - } - fn as_ingest(&self) -> Option<&dyn MemoryIngest> { - Some(self) - } - fn as_graph(&self) -> Option<&dyn MemoryGraph> { - Some(self) - } - fn as_goals(&self) -> Option<&dyn MemoryGoals> { - Some(self) - } - fn as_tool_memory(&self) -> Option<&dyn MemoryToolMemory> { - Some(self) - } - fn as_tree(&self) -> Option<&dyn MemoryTree> { - Some(self) - } - fn as_entities(&self) -> Option<&dyn MemoryEntities> { - Some(self) - } - fn as_diff(&self) -> Option<&dyn MemoryDiff> { - Some(self) - } - fn as_sources(&self) -> Option<&dyn MemorySourceSink> { - Some(self) - } - fn as_maintenance(&self) -> Option<&dyn MemoryMaintenance> { - Some(self) - } - fn as_people(&self) -> Option<&dyn MemoryPeople> { - Some(self) - } - fn as_chunks(&self) -> Option<&dyn MemoryChunks> { - Some(self) - } - fn as_retrieval(&self) -> Option<&dyn MemoryRetrieval> { - Some(self) - } - fn as_profile(&self) -> Option<&dyn MemoryProfile> { - Some(self) - } - fn as_episodic(&self) -> Option<&dyn MemoryEpisodic> { - Some(self) - } -} - -// ── People ─────────────────────────────────────────────────────────────────── -// -// The conversions below destructure both sides exhaustively rather than -// round-tripping through `Self::cross`. That is deliberate. `cross` is a serde -// value round-trip, so it agrees only while the two crates' field *names* agree -// — and they already do not: the engine's `Interaction` names its timestamp -// `ts` where the contract names it `at`. A round-trip would compile and then -// fail at runtime on the first call. -// -// Destructuring makes the opposite trade: a field added or renamed on either -// side is a compile error here, which is the same rule -// `tinymemory-tinycortex::convert` follows and the same reasoning that governs -// the two copies of the contract itself. - -/// The engine's people store for this module's workspace. -/// -/// `for_workspace` caches per workspace directory, so this is a map lookup -/// after the first call rather than a database open. -fn people_store( - workspace: &std::path::Path, -) -> Result, MemoryError> { - tinycortex::memory::people::store::for_workspace(workspace) - .map_err(|error| MemoryError::Other(anyhow::anyhow!("open people store: {error}"))) -} - -fn handle_to_engine(handle: &PersonHandle) -> tinycortex::memory::people::types::Handle { - use tinycortex::memory::people::types::Handle as EngineHandle; - match handle { - PersonHandle::IMessage(value) => EngineHandle::IMessage(value.clone()), - PersonHandle::Email(value) => EngineHandle::Email(value.clone()), - PersonHandle::DisplayName(value) => EngineHandle::DisplayName(value.clone()), - } -} - -fn handle_to_contract(handle: tinycortex::memory::people::types::Handle) -> PersonHandle { - use tinycortex::memory::people::types::Handle as EngineHandle; - match handle { - EngineHandle::IMessage(value) => PersonHandle::IMessage(value), - EngineHandle::Email(value) => PersonHandle::Email(value), - EngineHandle::DisplayName(value) => PersonHandle::DisplayName(value), - } -} - -fn person_to_contract(person: tinycortex::memory::people::types::Person) -> PersonRecord { - let tinycortex::memory::people::types::Person { - id, - display_name, - primary_email, - primary_phone, - handles, - created_at, - updated_at, - } = person; - PersonRecord { - id: id.to_string(), - display_name, - primary_email, - primary_phone, - handles: handles.into_iter().map(handle_to_contract).collect(), - created_at: created_at.to_rfc3339(), - updated_at: updated_at.to_rfc3339(), - } -} - -fn score_to_contract( - score: tinycortex::memory::people::types::ScoreComponents, - interaction_count: usize, -) -> PersonScore { - let tinycortex::memory::people::types::ScoreComponents { - recency, - frequency, - reciprocity, - depth, - score, - } = score; - PersonScore { - recency, - frequency, - reciprocity, - depth, - score, - interaction_count, - } -} - -/// Parse a caller-supplied person id. -/// -/// `PersonRef` is opaque to the caller by contract, so an unparseable one is a -/// caller mistake — `Invalid`, not `NotFound`. Reporting `NotFound` would tell -/// a caller the id was well-formed but absent, which would send them looking -/// for a deleted person rather than at the id they built. -fn parse_person_id( - person_id: &str, -) -> Result { - person_id - .parse::() - .map(tinycortex::memory::people::types::PersonId) - .map_err(|_| MemoryError::Invalid(format!("malformed person id: {person_id}"))) -} - -#[async_trait] -impl MemoryPeople for ModuleMemoryProvider { - async fn list_people(&self, limit: Option) -> Result, MemoryError> { - let store = people_store(&self.config.workspace_dir)?; - let people = store - .list() - .await - .map_err(|error| Self::other("list people", error))?; - - let ids: Vec<_> = people.iter().map(|person| person.id).collect(); - let interactions = store - .batch_interactions_for(&ids) - .await - .map_err(|error| Self::other("load interactions", error))?; - - let now = Utc::now(); - let mut ranked: Vec = people - .into_iter() - .map(|person| { - let observed = interactions.get(&person.id).map_or(&[][..], Vec::as_slice); - let closeness = tinycortex::memory::people::scorer::score(observed, now); - RankedPerson { - person: person_to_contract(person), - score: score_to_contract(closeness, observed.len()), - } - }) - .collect(); - - // Descending by composite score. `total_cmp` rather than `partial_cmp`: - // a NaN from a degenerate score would make `partial_cmp` return `None`, - // and an ordering that is not total is undefined behaviour's - // well-behaved cousin — `sort_by` may panic or produce garbage order. - ranked.sort_by(|a, b| b.score.score.total_cmp(&a.score.score)); - if let Some(limit) = limit { - ranked.truncate(limit); - } - Ok(ranked) - } - - async fn get_person(&self, person_id: &str) -> Result, MemoryError> { - let store = people_store(&self.config.workspace_dir)?; - let id = parse_person_id(person_id)?; - Ok(store - .get(id) - .await - .map_err(|error| Self::other("get person", error))? - .map(person_to_contract)) - } - - async fn resolve_handle( - &self, - handle: &PersonHandle, - create_if_missing: bool, - ) -> Result, MemoryError> { - let store = people_store(&self.config.workspace_dir)?; - let resolver = tinycortex::memory::people::resolver::HandleResolver::new(&store); - let engine_handle = handle_to_engine(handle); - - if create_if_missing { - let (id, created) = resolver - .resolve_or_create_with_status(&engine_handle) - .await - .map_err(|error| Self::other("resolve or create handle", error))?; - return Ok(Some(ResolvedPerson { - id: id.to_string(), - created, - })); - } - - Ok(resolver - .resolve(&engine_handle) - .await - .map_err(|error| Self::other("resolve handle", error))? - .map(|id| ResolvedPerson { - id: id.to_string(), - created: false, - })) - } - - async fn add_handle_alias( - &self, - person_id: &str, - handle: &PersonHandle, - ) -> Result<(), MemoryError> { - let store = people_store(&self.config.workspace_dir)?; - let id = parse_person_id(person_id)?; - if store - .get(id) - .await - .map_err(|error| Self::other("look up person", error))? - .is_none() - { - return Err(MemoryError::NotFound(format!("person {person_id}"))); - } - store - .add_alias(id, handle_to_engine(handle).canonicalize()) - .await - .map_err(|error| Self::other("add handle alias", error)) - } - - async fn score_person(&self, person_id: &str) -> Result, MemoryError> { - let store = people_store(&self.config.workspace_dir)?; - let id = parse_person_id(person_id)?; - if store - .get(id) - .await - .map_err(|error| Self::other("look up person", error))? - .is_none() - { - return Ok(None); - } - let interactions = store - .interactions_for(id) - .await - .map_err(|error| Self::other("load interactions", error))?; - Ok(Some(score_to_contract( - tinycortex::memory::people::scorer::score(&interactions, Utc::now()), - interactions.len(), - ))) - } - - async fn record_interaction(&self, interaction: &PersonInteraction) -> Result<(), MemoryError> { - let store = people_store(&self.config.workspace_dir)?; - let PersonInteraction { - person_id, - at, - is_outbound, - length, - } = interaction; - let id = parse_person_id(person_id)?; - let ts = chrono::DateTime::parse_from_rfc3339(at) - .map_err(|error| MemoryError::Invalid(format!("malformed interaction time: {error}")))? - .with_timezone(&Utc); - if store - .get(id) - .await - .map_err(|error| Self::other("look up person", error))? - .is_none() - { - return Err(MemoryError::NotFound(format!("person {person_id}"))); - } - store - .record_interaction(tinycortex::memory::people::types::Interaction { - person_id: id, - ts, - is_outbound: *is_outbound, - length: *length, - }) - .await - .map_err(|error| Self::other("record interaction", error)) - } - - async fn seed_from_address_book(&self) -> Result { - let store = people_store(&self.config.workspace_dir)?; - let resolver = tinycortex::memory::people::resolver::HandleResolver::new(&store); - let source = tinycortex::memory::people::address_book::SystemContactsSource; - let (seeded, skipped) = resolver - .seed_from_address_book(&source) - .await - .map_err(|error| Self::other("seed from address book", error))?; - Ok(AddressBookSeedOutcome { seeded, skipped }) - } -} - -// ── Chunks and Retrieval ───────────────────────────────────────────────────── -// -// Both families take the source scope as an **argument** and never read the -// ambient one. `tinymemory_core`'s in-process entry points resolve it from a -// task-local, which the host sets on its own side of the bus — it is simply not -// present in this process. Reading it here would yield `None`, and `None` means -// *unrestricted*, so a per-profile source gate would fail open. That is why the -// `*_scoped` variants exist and why these call them. - -/// Convert a contract scope into the engine's allowlist form. -fn scope_to_engine(scope: Option<&SourceScope>) -> Option> { - scope.map(|scope| scope.allow.iter().cloned().collect()) -} - -#[async_trait] -impl MemoryChunks for ModuleMemoryProvider { - async fn list_chunks( - &self, - query: &ChunkQuery, - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - let ChunkQuery { - source_kind, - source_id, - owner, - since_ms, - until_ms, - limit, - offset, - exclude_dropped, - } = query.clone(); - let engine_query = tinymemory_core::store::chunks::ListChunksQuery { - source_kind: source_kind - .map(|kind| Self::cross(&kind, "convert source kind")) - .transpose()?, - source_id, - owner, - since_ms, - until_ms, - limit, - offset, - source_scope: scope_to_engine(scope), - exclude_dropped, - }; - let chunks = blocking(self.config.clone(), "list chunks", move |config| { - tinymemory_core::store::chunks::list_chunks(config, &engine_query) - }) - .await?; - Self::cross(&chunks, "convert chunks") - } - - async fn get_chunk(&self, chunk_id: &str) -> Result, MemoryError> { - let id = chunk_id.to_string(); - let chunk = blocking(self.config.clone(), "get chunk", move |config| { - tinymemory_core::store::chunks::get_chunk(config, &id) - }) - .await?; - match chunk { - Some(chunk) => Ok(Some(Self::cross(&chunk, "convert chunk")?)), - None => Ok(None), - } - } - - async fn chunk_detail(&self, chunk_id: &str) -> Result, MemoryError> { - let id = chunk_id.to_string(); - let detail = blocking(self.config.clone(), "chunk detail", move |config| { - let Some(chunk) = tinymemory_core::store::chunks::get_chunk(config, &id)? else { - return Ok(None); - }; - // The vault read is best-effort: a missing body is reported as - // `None` so the caller can fall back to the row's own content, - // rather than failing the whole detail view over a preview. - let body = tinymemory_core::store::content::read::read_chunk_body(config, &id).ok(); - let has_embedding = - tinymemory_core::store::chunks::get_chunk_embedding(config, &id)?.is_some(); - let lifecycle_status = - tinymemory_core::store::chunks::get_chunk_lifecycle_status(config, &id)?; - let content_path = tinymemory_core::store::chunks::get_chunk_content_path(config, &id)?; - Ok(Some(( - chunk, - body, - has_embedding, - lifecycle_status, - content_path, - ))) - }) - .await?; - - let Some((chunk, body, has_embedding, lifecycle_status, content_path)) = detail else { - return Ok(None); - }; - Ok(Some(ChunkDetail { - chunk: Self::cross(&chunk, "convert chunk")?, - body, - content_path, - lifecycle_status, - has_embedding, - })) - } - - async fn storage_kinds(&self) -> Result, MemoryError> { - Ok(tinymemory_core::store::MemoryKind::ALL - .iter() - .map(|kind| kind.as_str().to_string()) - .collect()) - } - - async fn chunk_embeddings( - &self, - chunk_ids: &[String], - model_signature: &str, - ) -> Result, MemoryError> { - let ids = chunk_ids.to_vec(); - let signature = model_signature.to_string(); - let vectors = blocking( - self.config.clone(), - "load chunk embeddings", - move |config| { - tinymemory_core::store::chunks::get_chunk_embeddings_for_signature_batch( - config, &ids, &signature, - ) - }, - ) - .await?; - // Sorted so the response is deterministic: the engine returns a - // `HashMap`, whose iteration order varies per process and would make an - // otherwise-identical call return a differently-ordered list. - let mut embeddings: Vec = vectors - .into_iter() - .map(|(chunk_id, vector)| ChunkEmbedding { chunk_id, vector }) - .collect(); - embeddings.sort_by(|a, b| a.chunk_id.cmp(&b.chunk_id)); - Ok(embeddings) - } -} - -#[async_trait] -impl MemoryRetrieval for ModuleMemoryProvider { - async fn fast_retrieve( - &self, - query: &str, - options: FastRetrieveQuery, - scope: Option<&SourceScope>, - ) -> Result { - if query.trim().is_empty() { - return Err(MemoryError::Invalid("query must not be empty".to_string())); - } - let engine_options = tinymemory_core::tree::retrieval::FastRetrieveOptions { - limit: options.limit, - max_hops: options.max_hops, - time_window_days: options.time_window_days, - }; - let response = tinymemory_core::tree::retrieval::fast_retrieve_scoped( - &self.config, - query, - engine_options, - scope_to_engine(scope), - ) - .await - .map_err(|error| Self::other("fast retrieve", error))?; - Self::cross(&response, "convert retrieval response") - } - - async fn cover_window( - &self, - window: &CoverWindowQuery, - scope: Option<&SourceScope>, - ) -> Result { - let CoverWindowQuery { - since_ms, - until_ms, - source_id, - source_kind, - limit, - } = window.clone(); - let engine_kind = source_kind - .map(|kind| Self::cross(&kind, "convert source kind")) - .transpose()?; - let response = tinymemory_core::tree::retrieval::cover_window_scoped( - &self.config, - since_ms, - until_ms, - source_id.as_deref(), - engine_kind, - // 0 is the engine's "no caller preference" sentinel, not a request - // for zero rows: `cover_window_scoped` substitutes its own - // DEFAULT_LIMIT for it. Mapping `None` to 0 therefore asks for the - // default, which is what an absent limit means. - limit.unwrap_or(0), - scope_to_engine(scope), - ) - .await - .map_err(|error| Self::other("cover window", error))?; - Self::cross(&response, "convert retrieval response") - } - - async fn retrieve_source( - &self, - query: &SourceRetrievalQuery, - scope: Option<&SourceScope>, - ) -> Result { - let SourceRetrievalQuery { - source_id, - source_kind, - time_window_days, - query: text, - limit, - } = query.clone(); - let engine_kind = source_kind - .map(|kind| Self::cross(&kind, "convert source kind")) - .transpose()?; - let response = tinymemory_core::tree::retrieval::source::query_source_scoped( - &self.config, - tinymemory_core::tree::retrieval::source::SourceQuery { - source_id: source_id.as_deref(), - source_kind: engine_kind, - time_window_days, - query: text.as_deref(), - limit, - }, - scope_to_engine(scope), - ) - .await - .map_err(|error| Self::other("retrieve source", error))?; - Self::cross(&response, "convert retrieval response") - } - - async fn retrieve_children( - &self, - node_id: &str, - max_depth: u32, - query: Option<&str>, - limit: Option, - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - let hits = tinymemory_core::tree::retrieval::drill_down::drill_down_scoped( - &self.config, - node_id, - max_depth, - query, - limit, - scope_to_engine(scope), - ) - .await - .map_err(|error| Self::other("drill down", error))?; - Self::cross(&hits, "convert retrieval hits") - } - - async fn retrieve_leaves( - &self, - chunk_ids: &[String], - scope: Option<&SourceScope>, - ) -> Result, MemoryError> { - let hits = tinymemory_core::tree::retrieval::fetch::fetch_leaves_scoped( - &self.config, - chunk_ids, - scope_to_engine(scope), - ) - .await - .map_err(|error| Self::other("fetch leaves", error))?; - Self::cross(&hits, "convert retrieval hits") - } - - async fn recall_namespace_scored( - &self, - namespace: &str, - query: &str, - limit: usize, - exclude_session_id: Option<&str>, - ) -> Result, MemoryError> { - let hits = self - .client - .unified_handle() - .query_namespace_hits_excluding_session( - namespace, - query, - u32::try_from(limit).unwrap_or(u32::MAX), - exclude_session_id, - ) - .await - .map_err(|error| Self::other("recall namespace scored", error))?; - Self::cross(&hits, "convert namespace hits") - } - - async fn search_entities( - &self, - query: &str, - kinds: Option<&[String]>, - limit: usize, - ) -> Result, MemoryError> { - // Request kinds are validated, unlike response kinds which pass through - // as an open vocabulary. An unknown filter that silently matched nothing - // would be indistinguishable from a genuine empty result. - let engine_kinds = match kinds { - Some(kinds) => Some( - kinds - .iter() - .map(|kind| { - tinymemory_core::tree::score::extract::EntityKind::parse(kind).map_err( - |_| MemoryError::Invalid(format!("unknown entity kind: {kind}")), - ) - }) - .collect::, MemoryError>>()?, - ), - None => None, - }; - let matches = tinymemory_core::tree::retrieval::search_entities( - &self.config, - query, - engine_kinds, - limit, - ) - .await - .map_err(|error| Self::other("search entities", error))?; - Self::cross(&matches, "convert entity matches") - } -} - -// ── Profile ────────────────────────────────────────────────────────────────── -// -// `ProfileStore`'s methods are synchronous and hold a `parking_lot::Mutex` -// across a SQLite call, so each one goes through `spawn_blocking` rather than -// being awaited on the runtime thread. The store is cheap to obtain — it is a -// handle over the client's connection, not an open — so it is fetched inside -// the blocking closure rather than held across an await. - -fn facet_type_to_engine( - facet_type: FacetType, -) -> tinymemory_core::store::namespace_store::profile::FacetType { - use tinymemory_core::store::namespace_store::profile::FacetType as Engine; - match facet_type { - FacetType::Preference => Engine::Preference, - FacetType::Workflow => Engine::Workflow, - FacetType::Role => Engine::Role, - FacetType::Personality => Engine::Personality, - FacetType::Context => Engine::Context, - } -} - -#[async_trait] -impl MemoryProfile for ModuleMemoryProvider { - async fn list_active_facets(&self) -> Result, MemoryError> { - let client = Arc::clone(&self.client); - let facets = tokio::task::spawn_blocking(move || client.profile_store().list_active()) - .await - .map_err(|e| Self::other("join list_active_facets", e))? - .map_err(|e| Self::other("list_active_facets", e))?; - Self::cross(&facets, "convert facets") - } - - async fn list_all_facets(&self) -> Result, MemoryError> { - let client = Arc::clone(&self.client); - let facets = tokio::task::spawn_blocking(move || client.profile_store().list_all()) - .await - .map_err(|e| Self::other("join list_all_facets", e))? - .map_err(|e| Self::other("list_all_facets", e))?; - Self::cross(&facets, "convert facets") - } - - async fn get_facet(&self, key: &str) -> Result, MemoryError> { - let client = Arc::clone(&self.client); - let key = key.to_string(); - let facet = tokio::task::spawn_blocking(move || client.profile_store().get(&key)) - .await - .map_err(|e| Self::other("join get_facet", e))? - .map_err(|e| Self::other("get_facet", e))?; - match facet { - Some(facet) => Ok(Some(Self::cross(&facet, "convert facet")?)), - None => Ok(None), - } - } - - async fn facets_by_type( - &self, - facet_type: FacetType, - ) -> Result, MemoryError> { - let client = Arc::clone(&self.client); - let engine = facet_type_to_engine(facet_type); - let facets = - tokio::task::spawn_blocking(move || client.profile_store().facets_by_type(&engine)) - .await - .map_err(|e| Self::other("join facets_by_type", e))? - .map_err(|e| Self::other("facets_by_type", e))?; - Self::cross(&facets, "convert facets") - } - - async fn upsert_facet(&self, facet: &ProfileFacet) -> Result<(), MemoryError> { - let client = Arc::clone(&self.client); - let engine: tinymemory_core::store::namespace_store::profile::ProfileFacet = - Self::cross(facet, "convert facet")?; - tokio::task::spawn_blocking(move || client.profile_store().upsert_full(&engine)) - .await - .map_err(|e| Self::other("join upsert_facet", e))? - .map_err(|e| Self::other("upsert_facet", e)) - } - - async fn upsert_provider_facet( - &self, - facet_id: &str, - facet_type: FacetType, - key: &str, - value: &str, - confidence: f64, - segment_id: Option<&str>, - observed_at: f64, - ) -> Result<(), MemoryError> { - let client = Arc::clone(&self.client); - let engine = facet_type_to_engine(facet_type); - let (facet_id, key, value) = (facet_id.to_string(), key.to_string(), value.to_string()); - let segment_id = segment_id.map(str::to_string); - tokio::task::spawn_blocking(move || { - client.profile_store().upsert_provider_facet( - &facet_id, - &engine, - &key, - &value, - confidence, - segment_id.as_deref(), - observed_at, - ) - }) - .await - .map_err(|e| Self::other("join upsert_provider_facet", e))? - .map_err(|e| Self::other("upsert_provider_facet", e)) - } - - async fn set_facet_user_state( - &self, - key: &str, - user_state: UserState, - ) -> Result { - use tinymemory_core::store::namespace_store::profile::UserState as Engine; - let client = Arc::clone(&self.client); - let key = key.to_string(); - let engine = match user_state { - UserState::Auto => Engine::Auto, - UserState::Pinned => Engine::Pinned, - UserState::Forgotten => Engine::Forgotten, - }; - tokio::task::spawn_blocking(move || client.profile_store().set_user_state(&key, engine)) - .await - .map_err(|e| Self::other("join set_facet_user_state", e))? - .map_err(|e| Self::other("set_facet_user_state", e)) - } - - async fn delete_facet(&self, key: &str) -> Result { - let client = Arc::clone(&self.client); - let key = key.to_string(); - tokio::task::spawn_blocking(move || client.profile_store().delete(&key)) - .await - .map_err(|e| Self::other("join delete_facet", e))? - .map_err(|e| Self::other("delete_facet", e)) - } - - async fn delete_facet_by_id(&self, facet_id: &str) -> Result { - let client = Arc::clone(&self.client); - let facet_id = facet_id.to_string(); - tokio::task::spawn_blocking(move || client.profile_store().delete_by_facet_id(&facet_id)) - .await - .map_err(|e| Self::other("join delete_facet_by_id", e))? - .map_err(|e| Self::other("delete_facet_by_id", e)) - } - - async fn drop_facets_below(&self, threshold: f64) -> Result { - let client = Arc::clone(&self.client); - tokio::task::spawn_blocking(move || client.profile_store().drop_below_threshold(threshold)) - .await - .map_err(|e| Self::other("join drop_facets_below", e))? - .map_err(|e| Self::other("drop_facets_below", e)) - } - - async fn workflow_identity_matches(&self, key_pattern: &str, canonical_value: &str) -> bool { - let client = Arc::clone(&self.client); - let (pattern, value) = (key_pattern.to_string(), canonical_value.to_string()); - tokio::task::spawn_blocking(move || { - client - .profile_store() - .skill_identity_matches(&pattern, &value) - }) - .await - // A join failure reads as "no", like every other error on this - // predicate — see the trait docs. But it is logged first: the two - // cases behind it are a cancelled task and a panic inside - // `skill_identity_matches`, and a panic is a defect. Answering a bare - // `false` would make that defect look exactly like a legitimate - // non-match, which is the one reading that guarantees nobody - // investigates it. - .inspect_err(|error| { - log::error!( - "[tinymemory:module] workflow_identity_matches join failed, answering false: \ - {error}" - ); - }) - .unwrap_or(false) - } -} - -/// Episodic capture: the turn-by-turn record and its segment lifecycle. -/// -/// Every method hops to `spawn_blocking` for the same reason the profile family -/// does — these are synchronous `rusqlite` calls behind a `parking_lot::Mutex`, -/// and blocking a tinybus executor thread on a database lock would stall every -/// other call the module is serving. -/// -/// The boundary-detection and summary-composition halves of the archivist are -/// **not** here: they touch no database and are host policy. See the family's -/// contract docs. -#[async_trait] -impl MemoryEpisodic for ModuleMemoryProvider { - async fn insert_turn(&self, turn: &EpisodicTurn) -> Result { - let conn = self.client.profile_conn(); - let entry = tinymemory_core::store::fts5::EpisodicEntry { - id: None, - session_id: turn.session_id.clone(), - timestamp: turn.timestamp, - role: turn.role.clone(), - content: turn.content.clone(), - lesson: turn.lesson.clone(), - tool_calls_json: turn.tool_calls_json.clone(), - // The contract carries this signed because a cost is a plain number - // on the wire; the engine column is unsigned. A negative value is - // not meaningful, so it clamps rather than wrapping. - cost_microdollars: u64::try_from(turn.cost_microdollars).unwrap_or(0), - }; - tokio::task::spawn_blocking(move || { - tinymemory_core::store::fts5::episodic_insert(&conn, &entry) - }) - .await - .map_err(|e| Self::other("join insert_turn", e))? - .map_err(|e| Self::other("insert_turn", e)) - } - - async fn session_turns(&self, session_id: &str) -> Result, MemoryError> { - let conn = self.client.profile_conn(); - let session_id = session_id.to_string(); - let entries = tokio::task::spawn_blocking(move || { - tinymemory_core::store::fts5::episodic_session_entries(&conn, &session_id) - }) - .await - .map_err(|e| Self::other("join session_turns", e))? - .map_err(|e| Self::other("session_turns", e))?; - Ok(entries.into_iter().map(episodic_to_contract).collect()) - } - - async fn open_segment( - &self, - session_id: &str, - ) -> Result, MemoryError> { - let conn = self.client.profile_conn(); - let session_id = session_id.to_string(); - let segment = tokio::task::spawn_blocking(move || { - tinymemory_core::store::segments::open_segment_for_session(&conn, &session_id) - }) - .await - .map_err(|e| Self::other("join open_segment", e))? - .map_err(|e| Self::other("open_segment", e))?; - Ok(segment.map(segment_to_contract)) - } - - async fn create_segment( - &self, - segment_id: &str, - session_id: &str, - namespace: &str, - start_episodic_id: i64, - start_timestamp: f64, - now: f64, - ) -> Result<(), MemoryError> { - let conn = self.client.profile_conn(); - let (segment_id, session_id, namespace) = ( - segment_id.to_string(), - session_id.to_string(), - namespace.to_string(), - ); - tokio::task::spawn_blocking(move || { - tinymemory_core::store::segments::segment_create( - &conn, - &segment_id, - &session_id, - &namespace, - start_episodic_id, - // Per-session seq numbering is the archivist store's, and it is - // not part of this contract; legacy rows carry `None` too. - None, - start_timestamp, - now, - ) - }) - .await - .map_err(|e| Self::other("join create_segment", e))? - .map_err(|e| Self::other("create_segment", e)) - } - - async fn append_turn( - &self, - segment_id: &str, - episodic_id: i64, - timestamp: f64, - now: f64, - ) -> Result<(), MemoryError> { - let conn = self.client.profile_conn(); - let segment_id = segment_id.to_string(); - tokio::task::spawn_blocking(move || { - tinymemory_core::store::segments::segment_append_turn( - &conn, - &segment_id, - episodic_id, - None, - timestamp, - now, - ) - }) - .await - .map_err(|e| Self::other("join append_turn", e))? - .map_err(|e| Self::other("append_turn", e)) - } - - async fn close_segment(&self, segment_id: &str, now: f64) -> Result<(), MemoryError> { - let conn = self.client.profile_conn(); - let segment_id = segment_id.to_string(); - tokio::task::spawn_blocking(move || { - tinymemory_core::store::segments::segment_close(&conn, &segment_id, now) - }) - .await - .map_err(|e| Self::other("join close_segment", e))? - .map_err(|e| Self::other("close_segment", e)) - } - - async fn set_segment_summary( - &self, - segment_id: &str, - summary: &str, - now: f64, - ) -> Result<(), MemoryError> { - let conn = self.client.profile_conn(); - let (segment_id, summary) = (segment_id.to_string(), summary.to_string()); - tokio::task::spawn_blocking(move || { - tinymemory_core::store::segments::segment_set_summary(&conn, &segment_id, &summary, now) - }) - .await - .map_err(|e| Self::other("join set_segment_summary", e))? - .map_err(|e| Self::other("set_segment_summary", e)) - } - - async fn upsert_segment_embedding( - &self, - segment_id: &str, - model_signature: &str, - embedding: &[f32], - created_at: f64, - ) -> Result<(), MemoryError> { - let conn = self.client.profile_conn(); - let (segment_id, model_signature) = (segment_id.to_string(), model_signature.to_string()); - let embedding = embedding.to_vec(); - tokio::task::spawn_blocking(move || { - tinymemory_core::store::segments::segment_embedding_upsert( - &conn, - &segment_id, - &model_signature, - &embedding, - created_at, - ) - }) - .await - .map_err(|e| Self::other("join upsert_segment_embedding", e))? - .map_err(|e| Self::other("upsert_segment_embedding", e)) - } -} - -/// Engine episodic row -> contract turn. -fn episodic_to_contract(entry: tinymemory_core::store::fts5::EpisodicEntry) -> EpisodicTurn { - EpisodicTurn { - id: entry.id, - session_id: entry.session_id, - timestamp: entry.timestamp, - role: entry.role, - content: entry.content, - lesson: entry.lesson, - tool_calls_json: entry.tool_calls_json, - cost_microdollars: i64::try_from(entry.cost_microdollars).unwrap_or(i64::MAX), - } -} - -/// Engine segment row -> contract segment. -/// -/// Written out rather than derived: the engine row carries several fields the -/// contract deliberately does not expose (`topic_keywords`, the seq numbers, -/// `created_at`), and a blanket conversion would quietly start shipping them if -/// the contract ever grew a matching name. -fn segment_to_contract( - segment: tinymemory_core::store::segments::ConversationSegment, -) -> ConversationSegment { - use tinymemory_core::store::segments::SegmentStatus; - ConversationSegment { - segment_id: segment.segment_id, - session_id: segment.session_id, - namespace: segment.namespace, - start_episodic_id: segment.start_episodic_id, - end_episodic_id: segment.end_episodic_id, - start_timestamp: segment.start_timestamp, - end_timestamp: segment.end_timestamp, - turn_count: segment.turn_count, - summary: segment.summary, - embedding: segment.embedding, - open: matches!(segment.status, SegmentStatus::Open), - } +/// Builds the engine provider this module serves over the bus. +pub(crate) fn provider(config: &ModuleConfig, client: Arc) -> TinycortexProvider { + TinycortexProvider::new( + config.driver_id.clone(), + EngineRuntimeConfig::from(config), + client, + ) } diff --git a/crates/tinymemory-module/src/service/mod.rs b/crates/tinymemory-module/src/service/mod.rs index d959eaf..eba50ab 100644 --- a/crates/tinymemory-module/src/service/mod.rs +++ b/crates/tinymemory-module/src/service/mod.rs @@ -384,7 +384,7 @@ impl MemoryService { } })?; - let provider = crate::provider::ModuleMemoryProvider::new(&opener.config, Arc::new(client)); + let provider = crate::provider::provider(&opener.config, Arc::new(client)); opener .connection .serve_at( diff --git a/examples/basic.rs b/examples/basic.rs new file mode 100644 index 0000000..86e8951 --- /dev/null +++ b/examples/basic.rs @@ -0,0 +1,83 @@ +//! Bind a memory driver the way a host does: admit, then construct, then use. +//! +//! Run with: +//! +//! ```sh +//! cargo run --example basic +//! ``` +//! +//! This uses the null driver so it needs no engine, no workspace, and no +//! network — the point is the *shape* of binding, which is identical for a real +//! engine. Swap `NullMemoryProvider` for an adapter's provider and nothing else +//! here changes. +//! +//! The order matters and is the reason this example exists. A host does not +//! construct a driver and then ask whether it was allowed; it admits an id +//! first, and only then builds the thing. Admission is engine-neutral and +//! answers one question — *is this driver id real, and may it answer for +//! memory* — while construction needs everything an engine needs. + +use std::sync::Arc; + +use tinymemory::api::null::NullMemoryProvider; +use tinymemory::api::provider::{audit_provider, MemoryProvider}; +use tinymemory::api::types::{MemoryCategory, MemoryTaint, GLOBAL_NAMESPACE}; +use tinymemory::registry::{ConfigLabels, DriverRegistry, NULL_DRIVER_ID}; +use tinymemory::CONTRACT_VERSION; + +#[tokio::main] +async fn main() -> Result<(), Box> { + println!("contract version: {CONTRACT_VERSION:?}"); + + // 1. Admission. The host names a driver; the registry decides whether it is + // real and what class it binds as. A reserved embedded or null id needs + // no configuration entry, which is what lets an unconfigured host boot. + let registry = DriverRegistry::builtin(); + let admission = registry.admit(NULL_DRIVER_ID, None, ConfigLabels::default())?; + println!("admitted '{}' as {:?}", admission.id, admission.class); + + // 2. Construction. The host's job, not the registry's — see + // `tinymemory::registry`'s module docs for why the two are separate. + let provider: Arc = Arc::new(NullMemoryProvider::new()); + + // 3. Negotiation. `audit_provider` checks the driver advertises exactly the + // families it can actually serve. A driver whose capability set overstates + // its accessors would let a host register RPC methods that answer errors. + audit_provider(provider.as_ref())?; + // `Capabilities` is a set, not a string — render it by walking it, which is + // also how a host filters its RPC surface from the negotiated set. + let families: Vec<&str> = provider + .capabilities() + .iter() + .map(tinymemory::capabilities::Capability::as_str) + .collect(); + println!( + "driver '{}' serves {} families: {}", + provider.driver_id(), + families.len(), + families.join(", ") + ); + + // 4. Use. Every driver serves the three mandatory families, so this much + // works against any of them. + provider + .store( + GLOBAL_NAMESPACE, + "greeting", + "hello from the basic example", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await?; + + // The null driver accepts writes and discards them — `/dev/null` semantics, + // a legitimate binding for a deployment that wants the ports wired and + // nothing retained. Reading back nothing here is correct, not a failure. + match provider.get(GLOBAL_NAMESPACE, "greeting").await? { + Some(entry) => println!("read back: {}", entry.content), + None => println!("read back: nothing — the null driver retains no writes"), + } + + Ok(()) +} diff --git a/rust_out b/rust_out deleted file mode 100755 index 46d1184..0000000 Binary files a/rust_out and /dev/null differ diff --git a/src/registry/mod.rs b/src/registry/mod.rs index c41bd82..b5de045 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -40,6 +40,8 @@ use std::collections::BTreeMap; use std::fmt; +use tinymemory_api::host::MemoryHostConfig; + mod class; pub use class::{DriverClass, DriverClassParseError}; @@ -84,6 +86,16 @@ pub struct FallbackReason { pub reason: String, } +/// A refusal is an error, so `?` can propagate it. +/// +/// It carried `Display` from the start but not this, which meant a host writing +/// the obvious `registry.admit(..)?` in a function returning `Box` or +/// `anyhow::Error` got a type error instead. Nothing about the type changes — +/// this is the trait that makes the existing message usable where refusals +/// actually travel. Found by writing `examples/basic.rs` (issue #18 §E7), which +/// is the argument for having a compiled example at all. +impl std::error::Error for FallbackReason {} + impl fmt::Display for FallbackReason { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( @@ -284,6 +296,43 @@ impl DriverRegistry { Ok(admission) } + /// Selects and admits the memory driver this host's configuration names. + /// + /// The half of driver binding that was specified but never wired: the + /// registry could answer "is this driver id real and allowed", and nothing + /// asked it. This reads the id from the host's own configuration and puts + /// it through [`Self::admit`], so configuration decides the engine instead + /// of a factory hardcoding one (issue #18 §A5). + /// + /// It reads [`MemoryHostConfig::memory_driver`], **not** + /// `memory_provider` — despite the name, the latter is a `provider:model` + /// routing string choosing which language model does summarisation, which + /// is a different axis from which store the memory lives in. + /// + /// A configuration that names no driver gets [`TINYCORTEX_DRIVER_ID`], the + /// reserved embedded default. That is what keeps an unconfigured host + /// booting: an embedded id is admitted without a `drivers` entry, while an + /// external one is refused without endpoint, credential and trust + /// configuration. + /// + /// This resolves the *decision*, not the instance. Constructing the + /// provider, caching it per workspace, and wrapping it in a policy guard + /// stay with the host — see the module docs for why. + /// + /// # Errors + /// + /// Returns the [`FallbackReason`] to record and publish when the configured + /// driver is refused, exactly as [`Self::admit`] does. + pub fn select( + &self, + config: &dyn MemoryHostConfig, + entry: Option>, + labels: ConfigLabels<'_>, + ) -> Result { + let driver = config.memory_driver().unwrap_or(TINYCORTEX_DRIVER_ID); + self.admit(driver, entry, labels) + } + /// The class an id implies when nothing says otherwise. /// /// `context` names which part of the config was missing; the refusal echoes diff --git a/src/registry/test.rs b/src/registry/test.rs index 6d193e9..ab0186d 100644 --- a/src/registry/test.rs +++ b/src/registry/test.rs @@ -259,3 +259,77 @@ fn driver_class_serde_matches_the_config_spelling() { assert_eq!(json, format!("\"{}\"", class.as_str())); } } + +// ── Selection from configuration (issue #18 §A5) ───────────────────────────── +// +// Before this, the registry could answer "is this driver id real and allowed" +// and nothing asked it: `admit` had no production caller, and the memory client +// factory constructed TinyCortex unconditionally. These pin the wiring. + +use tinymemory_api::host::test_support::TestHostConfig; + +fn config_naming(driver: Option<&str>) -> TestHostConfig { + // `TestHostConfig` is `#[non_exhaustive]`, so it is built and then mutated + // rather than named field-by-field — which is what its own docs ask for. + let mut config = TestHostConfig::default(); + config.memory_driver = driver.map(str::to_owned); + config +} + +#[test] +fn a_configuration_naming_no_driver_gets_the_embedded_default() { + // The property that keeps an unconfigured host booting: a reserved embedded + // id is admitted without any `drivers` entry. + let admission = DriverRegistry::builtin() + .select(&config_naming(None), None, labels()) + .expect("an unconfigured host still binds"); + assert_eq!(admission.id, TINYCORTEX_DRIVER_ID); + assert_eq!(admission.class, DriverClass::Embedded); +} + +#[test] +fn a_configuration_naming_an_engine_selects_that_engine() { + let admission = DriverRegistry::builtin() + .select(&config_naming(Some(NULL_DRIVER_ID)), None, labels()) + .expect("the null driver is admitted without an entry"); + assert_eq!(admission.id, NULL_DRIVER_ID); + assert_eq!(admission.class, DriverClass::Null); +} + +#[test] +fn selecting_a_hosted_engine_still_requires_its_entry() { + // Selection does not loosen admission: an external driver named in config + // but left unconfigured is refused fail-closed, exactly as `admit` refuses + // it directly. + let reason = DriverRegistry::builtin() + .select(&config_naming(Some("supermemory")), None, labels()) + .expect_err("an external driver with no entry must be refused"); + assert_eq!(reason.configured_driver, "supermemory"); +} + +#[test] +fn selecting_a_hosted_engine_succeeds_once_it_is_configured_and_trusted() { + let entry = DriverEntry { + class: None, + trust_state: TRUSTED, + }; + let admission = DriverRegistry::builtin() + .select(&config_naming(Some("supermemory")), Some(entry), labels()) + .expect("a configured, trusted external driver is admitted"); + assert_eq!(admission.class, DriverClass::External); +} + +#[test] +fn selection_reads_the_engine_field_and_not_the_model_routing_one() { + // `memory_provider` is a `provider:model` routing string choosing which + // language model does summarisation. Reading it here would let a model + // change repoint a company's storage, which is why selection has its own + // field. + let mut config = TestHostConfig::default(); + config.memory_provider = Some("ollama:llama3".to_owned()); + config.memory_driver = None; + let admission = DriverRegistry::builtin() + .select(&config, None, labels()) + .expect("model routing must not affect engine selection"); + assert_eq!(admission.id, TINYCORTEX_DRIVER_ID); +} diff --git a/tests/capability_negotiation.rs b/tests/capability_negotiation.rs new file mode 100644 index 0000000..c43081f --- /dev/null +++ b/tests/capability_negotiation.rs @@ -0,0 +1,197 @@ +//! Capability negotiation: what a host may trust a driver's advertisement for, +//! and what happens when the advertisement is wrong. +//! +//! The contract's premise is that a host negotiates once at bind time and then +//! filters its own surface from the cached set. That is only safe if the set is +//! honest, which is what `audit_provider` is for — so these tests pin both the +//! honest path and the dishonest one. + +// A failing assertion in a test *is* a panic; the crate-wide `expect_used` / +// `unwrap_used` / `panic` lints exist to keep the library from panicking, not +// the tests. Same allowance, and same reasoning, as `src/registry/test.rs`. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use std::sync::Arc; + +use tinymemory::api::capabilities::{Capabilities, Capability}; +use tinymemory::api::health::MemoryHealth; +use tinymemory::api::null::NullMemoryProvider; +use tinymemory::api::provider::{audit_provider, MemoryProvider, MemoryTree}; +use tinymemory_conformance::InMemoryProvider; + +#[test] +fn the_reference_drivers_advertise_exactly_what_they_reach() { + for provider in [ + Arc::new(InMemoryProvider::new()) as Arc, + Arc::new(NullMemoryProvider::new()), + ] { + assert!( + audit_provider(provider.as_ref()).is_ok(), + "driver `{}` failed its audit", + provider.driver_id() + ); + } +} + +#[test] +fn a_host_can_filter_its_surface_from_the_cached_capability_set() { + // This is the whole point of negotiating once: a host reads the set at bind + // time and never asks again, so the set has to answer both directions. + let provider = InMemoryProvider::new(); + let caps = provider.capabilities(); + + for mandatory in Capability::MANDATORY { + assert!( + caps.contains(mandatory), + "{} must be advertised", + mandatory.as_str() + ); + assert!( + provider.provides(mandatory), + "{} must be reachable", + mandatory.as_str() + ); + } + + // An optional family this driver does not serve is absent from the set AND + // unreachable through its accessor. A host that registered an RPC method + // from the set alone would otherwise expose a method that answers errors. + assert!(!caps.contains(Capability::Tree)); + assert!(provider.as_tree().is_none()); + assert!(!provider.provides(Capability::Tree)); +} + +/// A driver that claims a family it cannot serve. +/// +/// Exists to prove the audit catches it. This is the failure mode the audit was +/// written for: the claim is cheap to make and, without a check, only surfaces +/// on the first call — which for a memory family may be days later, on a path +/// nobody is watching. +#[derive(Debug, Default)] +struct LyingProvider(InMemoryProvider); + +#[async_trait::async_trait] +impl tinymemory::api::provider::MemoryCore for LyingProvider { + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: tinymemory::types::MemoryCategory, + session_id: Option<&str>, + taint: tinymemory::types::MemoryTaint, + ) -> Result<(), tinymemory::error::MemoryError> { + self.0 + .store(namespace, key, content, category, session_id, taint) + .await + } + async fn get( + &self, + namespace: &str, + key: &str, + ) -> Result, tinymemory::error::MemoryError> { + self.0.get(namespace, key).await + } + async fn forget( + &self, + namespace: &str, + key: &str, + ) -> Result { + self.0.forget(namespace, key).await + } + async fn list( + &self, + namespace: Option<&str>, + category: Option<&tinymemory::types::MemoryCategory>, + session_id: Option<&str>, + ) -> Result, tinymemory::error::MemoryError> { + self.0.list(namespace, category, session_id).await + } + async fn namespaces( + &self, + ) -> Result, tinymemory::error::MemoryError> { + self.0.namespaces().await + } +} + +#[async_trait::async_trait] +impl tinymemory::api::provider::MemoryRecall for LyingProvider { + async fn recall( + &self, + query: &str, + limit: usize, + opts: &tinymemory::recall::OwnedRecallOpts, + scope: Option<&tinymemory::api::provider::SourceScope>, + ) -> Result, tinymemory::error::MemoryError> { + self.0.recall(query, limit, opts, scope).await + } +} + +#[async_trait::async_trait] +impl tinymemory::api::provider::MemoryPortability for LyingProvider { + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.0.export_page(cursor, limit).await + } + async fn import_records( + &self, + records: Vec, + ) -> Result { + self.0.import_records(records).await + } +} + +#[async_trait::async_trait] +impl MemoryProvider for LyingProvider { + fn driver_id(&self) -> &'static str { + "liar" + } + + fn capabilities(&self) -> Capabilities { + // Claims a summary tree it has no accessor for. + Capabilities::mandatory().with(Capability::Tree) + } + + async fn health(&self) -> MemoryHealth { + MemoryHealth::Ready + } + + // `as_tree` deliberately left at its `None` default. +} + +#[test] +fn a_driver_that_advertises_a_family_it_cannot_serve_fails_the_audit() { + let liar = LyingProvider::default(); + let audit = audit_provider(&liar).expect_err("the audit must catch an overstated capability"); + assert!( + audit.advertised_but_absent.contains(&Capability::Tree), + "the audit should name the family: {audit:?}" + ); + assert!( + audit.present_but_unadvertised.is_empty(), + "nothing was under-advertised here: {audit:?}" + ); +} + +#[test] +fn the_audit_failure_renders_something_an_operator_can_act_on() { + let audit = audit_provider(&LyingProvider::default()) + .expect_err("the audit must fail") + .to_string(); + assert!( + audit.contains("tree"), + "the message should name the family: {audit}" + ); +} + +/// Compile-time proof that `as_tree` returning `Some` is what "reachable" +/// means, so the audit is checking the accessor and not a second declaration. +#[test] +fn reachability_is_the_accessor_not_a_second_declaration() { + let provider = InMemoryProvider::new(); + let tree: Option<&dyn MemoryTree> = provider.as_tree(); + assert!(tree.is_none()); +} diff --git a/tests/driver_selection.rs b/tests/driver_selection.rs new file mode 100644 index 0000000..ea3a36d --- /dev/null +++ b/tests/driver_selection.rs @@ -0,0 +1,230 @@ +//! Driver admission: which ids exist, what class each binds as, and what is +//! refused. +//! +//! Exercises only the public surface of the `tinymemory` facade. +//! +//! # Selection +//! +//! Configuration now chooses the engine (§A5). One correction to the issue is +//! worth recording here, because it would otherwise have wired the wrong thing: +//! §A5 names `MemoryHostConfig::memory_provider()` as the selector, but that +//! method is a `provider:model` routing string for the memory *workload* — +//! which language model summarises — not the engine the memory lives in. +//! Selection reads `memory_driver()` instead, added for the purpose. +//! +//! What still does not exist is the last clause of §A5, that `create_memory_*` +//! return a bound `Arc`. It cannot, and the reason is +//! structural rather than unfinished: `adapters/tinycortex` depends on +//! `tinymemory-core` since §C3, so a core factory returning a constructed +//! adapter provider would be a dependency cycle. Selection resolves the +//! *decision*; the host constructs — which is what `src/registry`'s own module +//! docs have always said. + +// A failing assertion in a test *is* a panic; the crate-wide `expect_used` / +// `unwrap_used` / `panic` lints exist to keep the library from panicking, not +// the tests. Same allowance, and same reasoning, as `src/registry/test.rs`. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use tinymemory::api::host::test_support::TestHostConfig; +use tinymemory::registry::{ + ConfigLabels, DriverClass, DriverEntry, DriverRegistry, COGNEE_DRIVER_ID, MEM0_DRIVER_ID, + SUPERMEMORY_DRIVER_ID, TINYCORTEX_DRIVER_ID, TRUSTED, +}; + +fn labels() -> ConfigLabels<'static> { + ConfigLabels { + section: "[memory]", + drivers: "[memory.drivers]", + driver_entry: "[memory.drivers.]", + } +} + +fn trusted_external() -> DriverEntry<'static> { + DriverEntry { + class: None, + trust_state: TRUSTED, + } +} + +#[test] +fn a_reserved_embedded_id_is_admitted_without_any_config_entry() { + // The embedded default's options live in the host's own config blocks, so + // it must not require a `drivers` entry to be selectable at all. + let admission = DriverRegistry::builtin() + .admit(TINYCORTEX_DRIVER_ID, None, labels()) + .expect("the built-in embedded engine is admitted"); + assert_eq!(admission.id, TINYCORTEX_DRIVER_ID); + assert_eq!(admission.class, DriverClass::Embedded); +} + +#[test] +fn the_null_driver_is_admitted_and_is_class_null() { + let admission = DriverRegistry::builtin() + .admit(tinymemory::registry::NULL_DRIVER_ID, None, labels()) + .expect("the null driver is admitted"); + assert_eq!(admission.class, DriverClass::Null); +} + +#[test] +fn every_reserved_external_id_resolves_to_the_external_class() { + let registry = DriverRegistry::builtin(); + for id in [SUPERMEMORY_DRIVER_ID, MEM0_DRIVER_ID, COGNEE_DRIVER_ID] { + let admission = registry + .admit(id, Some(trusted_external()), labels()) + .unwrap_or_else(|reason| panic!("{id} was refused: {}", reason.reason)); + assert_eq!(admission.class, DriverClass::External, "{id}"); + assert_eq!(admission.id, id); + } +} + +#[test] +fn an_external_driver_without_an_entry_is_refused_fail_closed() { + // The fail-closed half: an external engine needs endpoint, credential and + // trust configuration, so admitting it implicitly would bind an + // out-of-process backend nobody configured. + let reason = DriverRegistry::builtin() + .admit(SUPERMEMORY_DRIVER_ID, None, labels()) + .expect_err("an external driver with no entry must be refused"); + assert_eq!(reason.configured_driver, SUPERMEMORY_DRIVER_ID); + assert!( + reason.reason.contains("external"), + "the refusal should say why: {}", + reason.reason + ); +} + +#[test] +fn an_untrusted_external_driver_is_refused_even_with_an_entry() { + let entry = DriverEntry { + class: None, + trust_state: "untrusted", + }; + let reason = DriverRegistry::builtin() + .admit(SUPERMEMORY_DRIVER_ID, Some(entry), labels()) + .expect_err("trust must be raised explicitly before an external bind"); + assert!( + reason.reason.contains(TRUSTED), + "the refusal should name the value to set: {}", + reason.reason + ); +} + +#[test] +fn a_reserved_id_cannot_have_its_class_overridden_by_config() { + // A reserved id names a fixed implementation. An explicit `class` line may + // confirm it but never override it — otherwise config could run the + // embedded engine under the checks meant for an external one. + let entry = DriverEntry { + class: Some("external"), + trust_state: TRUSTED, + }; + let reason = DriverRegistry::builtin() + .admit(TINYCORTEX_DRIVER_ID, Some(entry), labels()) + .expect_err("a reserved id's class must not be overridable"); + assert!( + reason.reason.contains("built in"), + "the refusal should explain why: {}", + reason.reason + ); +} + +#[test] +fn an_unknown_driver_id_is_refused_rather_than_defaulted() { + let reason = DriverRegistry::builtin() + .admit("not-an-engine", None, labels()) + .expect_err("an unreserved id with no entry must be refused"); + assert_eq!(reason.configured_driver, "not-an-engine"); +} + +#[test] +fn an_empty_driver_id_is_refused() { + let reason = DriverRegistry::builtin() + .admit(" ", None, labels()) + .expect_err("a blank driver id must be refused"); + assert!( + reason.reason.contains("empty"), + "the refusal should name the problem: {}", + reason.reason + ); +} + +#[test] +fn a_config_class_typo_is_echoed_back_to_the_operator() { + // The offending value comes from the host's own config file, not from a + // driver or the network, so echoing it discloses nothing the reader did not + // write — and without it the message cannot point at the line to fix. + let entry = DriverEntry { + class: Some("embeded"), + trust_state: TRUSTED, + }; + let reason = DriverRegistry::builtin() + .admit("some-driver", Some(entry), labels()) + .expect_err("an unparseable class must be refused"); + assert!( + reason.reason.contains("embeded"), + "the refusal should quote the typo: {}", + reason.reason + ); +} + +// ── The selection half, through the public facade ──────────────────────────── + +fn config_naming(driver: Option<&str>) -> TestHostConfig { + let mut config = TestHostConfig::default(); + config.memory_driver = driver.map(str::to_owned); + config +} + +#[test] +fn configuration_chooses_the_engine_and_admission_gates_it() { + let admission = DriverRegistry::builtin() + .select( + &config_naming(Some(COGNEE_DRIVER_ID)), + Some(trusted_external()), + labels(), + ) + .expect("a configured, trusted external engine binds"); + assert_eq!(admission.id, COGNEE_DRIVER_ID); + assert_eq!(admission.class, DriverClass::External); +} + +#[test] +fn an_unconfigured_host_still_binds_the_embedded_default() { + // The property that matters most operationally: adding engine selection + // must not turn "I configured nothing" into a host that fails to start. + let admission = DriverRegistry::builtin() + .select(&config_naming(None), None, labels()) + .expect("an unconfigured host binds the embedded default"); + assert_eq!(admission.id, TINYCORTEX_DRIVER_ID); + assert_eq!(admission.class, DriverClass::Embedded); +} + +#[test] +fn selection_does_not_loosen_the_fail_closed_external_gate() { + // Going through `select` rather than `admit` must not become a way around + // the trust requirement. + let untrusted = DriverEntry { + class: None, + trust_state: "untrusted", + }; + let reason = DriverRegistry::builtin() + .select( + &config_naming(Some(MEM0_DRIVER_ID)), + Some(untrusted), + labels(), + ) + .expect_err("an untrusted external engine is refused however it was chosen"); + assert!(reason.reason.contains(TRUSTED), "{}", reason.reason); +} + +#[test] +fn the_model_routing_field_cannot_repoint_the_store() { + // `memory_provider` chooses a language model; `memory_driver` chooses the + // store. Conflating them would let a model change move a company's memory. + let mut config = TestHostConfig::default(); + config.memory_provider = Some("ollama:llama3".to_owned()); + let admission = DriverRegistry::builtin() + .select(&config, None, labels()) + .expect("model routing leaves engine selection alone"); + assert_eq!(admission.id, TINYCORTEX_DRIVER_ID); +} diff --git a/tests/null_provider.rs b/tests/null_provider.rs new file mode 100644 index 0000000..83e3a8c --- /dev/null +++ b/tests/null_provider.rs @@ -0,0 +1,117 @@ +//! The `null` driver: the configuration a compiled-out or unconfigured memory +//! subsystem binds to. +//! +//! It has to be genuinely usable, not a placeholder that panics. A host whose +//! memory is switched off still calls the ports, and the difference between +//! "returns empty" and "aborts the process" is the difference between a +//! degraded deployment and an outage. + +// A failing assertion in a test *is* a panic; the crate-wide `expect_used` / +// `unwrap_used` / `panic` lints exist to keep the library from panicking, not +// the tests. Same allowance, and same reasoning, as `src/registry/test.rs`. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use std::sync::Arc; + +use tinymemory::api::capabilities::{Capabilities, Capability}; +use tinymemory::api::null::{NullMemoryProvider, NULL_DRIVER_ID}; +use tinymemory::api::provider::{audit_provider, MemoryProvider}; +use tinymemory::types::{MemoryCategory, MemoryTaint}; + +const NS: &str = "null-provider"; + +#[test] +fn it_identifies_itself_and_passes_its_own_audit() { + let provider = NullMemoryProvider::new(); + assert_eq!(provider.driver_id(), NULL_DRIVER_ID); + assert!(audit_provider(&provider).is_ok()); + assert_eq!(provider.capabilities(), Capabilities::mandatory()); +} + +#[tokio::test] +async fn every_mandatory_method_answers_rather_than_panicking() { + let provider: Arc = Arc::new(NullMemoryProvider::new()); + + provider + .store( + NS, + "k", + "v", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store is accepted and discarded, not refused"); + assert!(provider.get(NS, "k").await.expect("get answers").is_none()); + assert!(!provider.forget(NS, "k").await.expect("forget answers")); + assert!(provider + .list(None, None, None) + .await + .expect("list answers") + .is_empty()); + assert!(provider + .namespaces() + .await + .expect("namespaces answers") + .is_empty()); + + let opts = tinymemory::recall::OwnedRecallOpts::default(); + assert!(provider + .recall("anything", 10, &opts, None) + .await + .expect("recall answers") + .is_empty()); + + let page = provider + .export_page(None, 10) + .await + .expect("export answers"); + assert!(page.records.is_empty()); + assert!(page.next_cursor.is_none(), "an empty export must terminate"); + + let outcome = provider + .import_records(Vec::new()) + .await + .expect("import answers"); + assert_eq!(outcome.imported, 0); + assert_eq!(outcome.failed, 0); +} + +#[tokio::test] +async fn it_is_healthy_rather_than_reporting_a_fault() { + // "Memory is switched off" is a configuration, not a failure. Reporting + // unhealthy would make an intentional deployment look like a broken one. + let provider = NullMemoryProvider::new(); + assert_eq!( + provider.health().await, + tinymemory::health::MemoryHealth::Ready + ); +} + +#[test] +fn no_optional_family_is_reachable_and_none_is_advertised() { + let provider = NullMemoryProvider::new(); + for capability in Capability::ALL { + if Capability::MANDATORY.contains(&capability) { + continue; + } + assert!( + !provider.provides(capability), + "`{}` must not be reachable on the null driver", + capability.as_str() + ); + assert!( + !provider.capabilities().contains(capability), + "`{}` must not be advertised on the null driver", + capability.as_str() + ); + } +} + +#[tokio::test] +async fn it_conforms_to_the_behavioural_suite() { + // The contract-shape half of the suite applies to a discard driver exactly + // as it does to a retaining one; the suite skips only the storage half. + tinymemory_conformance::assert_provider(Arc::new(NullMemoryProvider::new())).await; +} diff --git a/tests/taint_end_to_end.rs b/tests/taint_end_to_end.rs new file mode 100644 index 0000000..1841fb4 --- /dev/null +++ b/tests/taint_end_to_end.rs @@ -0,0 +1,208 @@ +//! Provenance, end to end through the public surface. +//! +//! `MemoryTaint` decides whether downstream policy treats content as something +//! the user authored or as something that arrived from outside. A driver that +//! loses it does not fail loudly — it silently reclassifies external content as +//! internal-trust, and every gate keyed on taint is then wrong about everything +//! that passed through. +//! +//! # Scope note +//! +//! Issue #18 §E3 describes this file as asserting that "external content stored +//! through the **sync path** arrives with `ExternalSync` at every engine". The +//! sync layer is welded to the engine today (§1.4) and its rewrite onto the +//! memory API is §B, so there is no engine-neutral sync path to drive yet. +//! +//! What is assertable now is the seam sync will hand to: taint through store, +//! read-back, list, recall, and the export/import round trip. When §B lands, +//! the sync leg is added here rather than in a new file. + +// A failing assertion in a test *is* a panic; the crate-wide `expect_used` / +// `unwrap_used` / `panic` lints exist to keep the library from panicking, not +// the tests. Same allowance, and same reasoning, as `src/registry/test.rs`. +#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)] + +use std::sync::Arc; + +use tinymemory::api::null::NullMemoryProvider; +use tinymemory::api::provider::{MemoryCore, MemoryPortability, MemoryProvider, MemoryRecall}; +use tinymemory::types::{MemoryCategory, MemoryTaint}; +use tinymemory_conformance::InMemoryProvider; + +const NS: &str = "taint-e2e"; + +/// Every driver this workspace ships, so the assertion is "at every engine" +/// rather than "at the one we happened to test". +fn drivers() -> Vec> { + vec![ + Arc::new(InMemoryProvider::new()), + Arc::new(NullMemoryProvider::new()), + ] +} + +#[tokio::test] +async fn external_content_reads_back_as_external_at_every_driver() { + for provider in drivers() { + let who = provider.driver_id(); + provider + .store( + NS, + "from-the-web", + "scraped from a page", + MemoryCategory::Conversation, + None, + MemoryTaint::ExternalSync, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + + // A driver that retains nothing has nothing to reclassify; one that + // retains must hand back what it was given. + if let Some(entry) = provider.get(NS, "from-the-web").await.unwrap_or(None) { + assert_eq!( + entry.taint, + MemoryTaint::ExternalSync, + "{who}: external content was laundered into internal-trust content" + ); + } + let _ = provider.forget(NS, "from-the-web").await; + } +} + +#[tokio::test] +async fn internal_content_is_not_marked_external_by_accident() { + // The inverse error is just as bad in the other direction: over-marking + // makes the gate refuse the company's own material. + for provider in drivers() { + let who = provider.driver_id(); + provider + .store( + NS, + "our-own", + "we decided this", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .unwrap_or_else(|e| panic!("{who}: store failed: {e}")); + if let Some(entry) = provider.get(NS, "our-own").await.unwrap_or(None) { + assert_eq!( + entry.taint, + MemoryTaint::Internal, + "{who}: internal content was over-marked" + ); + } + let _ = provider.forget(NS, "our-own").await; + } +} + +#[tokio::test] +async fn taint_survives_list_and_recall_not_just_get() { + // `get` is the easy path. A driver that rebuilds entries on the list and + // recall paths can drop provenance on exactly those, which is where a + // policy gate actually reads it. + let provider = InMemoryProvider::new(); + provider + .store( + NS, + "k", + "needle from outside", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .expect("store"); + + let listed = provider.list(Some(NS), None, None).await.expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!( + listed[0].taint, + MemoryTaint::ExternalSync, + "list dropped provenance" + ); + + let opts = tinymemory::recall::OwnedRecallOpts { + namespace: Some(NS.to_string()), + ..Default::default() + }; + let hits = provider + .recall("needle", 10, &opts, None) + .await + .expect("recall"); + assert_eq!(hits.len(), 1); + assert_eq!( + hits[0].taint, + MemoryTaint::ExternalSync, + "recall dropped provenance" + ); +} + +#[tokio::test] +async fn taint_survives_export_and_re_import() { + // The migration case. An export that drops taint, or an import that + // re-stamps it, turns every restored external record into internal-trust + // content — and a restore is exactly when nobody is watching. + let provider = InMemoryProvider::new(); + provider + .store( + NS, + "moved", + "carried across", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .expect("store"); + + let page = provider.export_page(None, 64).await.expect("export"); + let record = page + .records + .iter() + .find(|r| r.namespace.as_deref() == Some(NS)) + .expect("the stored record was exported"); + assert_eq!( + record.taint, + MemoryTaint::ExternalSync, + "export dropped provenance" + ); + + let fresh = InMemoryProvider::new(); + let outcome = fresh + .import_records(vec![record.clone()]) + .await + .expect("import"); + assert_eq!(outcome.imported, 1); + assert_eq!(outcome.failed, 0, "{:?}", outcome.errors); + + let restored = fresh + .get(NS, "moved") + .await + .expect("get") + .expect("restored"); + assert_eq!( + restored.taint, + MemoryTaint::ExternalSync, + "import re-stamped provenance instead of persisting what it was given" + ); +} + +#[test] +fn unknown_persisted_taint_values_fail_closed() { + // A corrupt or future column value must read as the *more* restrictive + // state. Failing open here would let an unrecognised row be treated as + // user-authored, which is the one direction that cannot be undone. + assert_eq!(MemoryTaint::from_db_str(""), MemoryTaint::ExternalSync); + assert_eq!( + MemoryTaint::from_db_str("future-value"), + MemoryTaint::ExternalSync + ); + assert_eq!( + MemoryTaint::from_db_str("INTERNAL"), + MemoryTaint::ExternalSync + ); + // Only the exact known spelling reads as internal. + assert_eq!(MemoryTaint::from_db_str("internal"), MemoryTaint::Internal); +} diff --git a/tmp b/tmp deleted file mode 100755 index 9cd4ca7..0000000 Binary files a/tmp and /dev/null differ