From 8d9a24f7a9f6eda23c80f14b4c3c0656218d1208 Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Tue, 6 Oct 2026 00:24:47 +0530 Subject: [PATCH] fix(import): keep v1 taint as a taint:external_sync tag v1 stamped every memory_docs row with a taint (internal or external_sync) and kept externally synced content out of external-effect tool decisions. The importer documented the column as probed but never read it, so Gmail/Slack/Notion content synced in v1 arrived in v2 indistinguishable from the user's own memory. Probe memory_docs.taint and tag every item from a row whose taint is not `internal` with `taint:external_sync` (EXTERNAL_SYNC_TAG, exported), in documents, learnings and global rows. The decode fails closed like v1's own; a store from before the column reads as internal, as v1 read it. The tag rides in MemoryMeta, which the CortexDB engine stores whole in its envelope, so no engine change is needed. Refs tinyhumansai/openhuman#7005. --- .../src/import/README.md | 16 +++ .../tinymemory-integrations/src/import/mod.rs | 6 +- .../src/import/sections/memory_docs.rs | 38 +++++- .../src/import/sections/memory_docs_tests.rs | 10 ++ .../src/import/sections/mod.rs | 2 + .../src/import/workspace/schema.rs | 3 + .../tests/legacy_import.rs | 113 +++++++++++++++++- 7 files changed, 184 insertions(+), 4 deletions(-) diff --git a/crates/tinymemory-integrations/src/import/README.md b/crates/tinymemory-integrations/src/import/README.md index d7db2a73..847f3791 100644 --- a/crates/tinymemory-integrations/src/import/README.md +++ b/crates/tinymemory-integrations/src/import/README.md @@ -33,6 +33,7 @@ the importer reads. A missing path is `NotFound`; anything else that is not a v1 store (a file, no `memory.db`, a non-SQLite file, a different schema) is `NotLegacy` with the reason. Columns that later v1 migrations added are probed with `pragma_table_info` and used when present: `memory_docs.logical_namespace`, +`memory_docs.taint`, `episodic_log.tool_calls_json`, `user_profile.state` / `user_state` / `class` / `evidence_refs_json`, and `mem_tree_chunks.content_path`. @@ -73,6 +74,21 @@ restored). Then: Rows with blank content are skipped in every section. +### Taint + +v1 stamped every `memory_docs` row with a `taint`: `internal` for what the +user and the agent wrote, `external_sync` for content synced from an outside +service (Gmail, Slack, Notion, Composio, MCP, ...), and kept tainted content +out of decisions to call external-effect tools. v2 has no taint field, so an +item from a row whose `taint` is anything but `internal` gets the tag +`taint:external_sync` (`EXTERNAL_SYNC_TAG`), in every section a `memory_docs` +row can land in (documents, learnings, `global`). The tag rides in the item's +metadata, which a CortexDB engine stores whole, so the host can read it back on +recall. The decode fails closed like v1's: an unknown or empty value is +external. A store from before the `taint` column is read as all `internal`, +which is how v1 read it. `episodic_log`, `user_profile` and the chunk store +have no taint in v1 and get no tag. + ### Documents `title` from `title` (none when blank), body = `content`, `tags` = the strings diff --git a/crates/tinymemory-integrations/src/import/mod.rs b/crates/tinymemory-integrations/src/import/mod.rs index aa56c1a3..e5cd60a6 100644 --- a/crates/tinymemory-integrations/src/import/mod.rs +++ b/crates/tinymemory-integrations/src/import/mod.rs @@ -18,8 +18,9 @@ //! Every item's `meta.source` is `SourceKind::Import` with a section-scoped //! legacy id (`memory_docs:`, `episodic_log:`, //! `user_profile:`, `mem_tree_chunks::`), and -//! `meta.workspace` is the workspace path. The module's `README.md` details -//! every mapping decision. +//! `meta.workspace` is the workspace path. A `memory_docs` row v1 marked as +//! synced from an external service also carries [`EXTERNAL_SYNC_TAG`]. The +//! module's `README.md` details every mapping decision. //! //! Import is resumable: each [`ImportedItem`] carries the [`Checkpoint`] to //! persist once its item is stored, and [`LegacyWorkspace::items_from`] @@ -78,6 +79,7 @@ pub use checkpoint::{Checkpoint, ChunkCursor, ImportedItem}; pub use error::{Error, Result}; pub use items::{DEFAULT_PAGE_SIZE, Items}; pub use migrate::{MigrationReport, migrate, migrate_with}; +pub use sections::EXTERNAL_SYNC_TAG; pub use workspace::LegacyWorkspace; /// Re-exported so a host names the same item type the importer yields. diff --git a/crates/tinymemory-integrations/src/import/sections/memory_docs.rs b/crates/tinymemory-integrations/src/import/sections/memory_docs.rs index bfb468a3..13a52241 100644 --- a/crates/tinymemory-integrations/src/import/sections/memory_docs.rs +++ b/crates/tinymemory-integrations/src/import/sections/memory_docs.rs @@ -14,6 +14,12 @@ //! //! Both sections scan the whole table by `document_id` and skip the rows that //! belong to the other one. +//! +//! A row v1 marked as synced from an external service (`taint` other than +//! `internal`) is tagged [`EXTERNAL_SYNC_TAG`], whichever section it lands in, +//! so the host can keep treating it as untrusted. The decode fails closed like +//! v1's own: an unknown or empty value is external. A store from before the +//! `taint` column has none to read, and v1 read those rows as internal. use rusqlite::params; use serde_json::Value; @@ -37,12 +43,19 @@ const SECTION_PREFIXES: [&str; 9] = [ "event", ]; +/// The tag on every item from a `memory_docs` row v1 marked as synced from an +/// external service (Gmail, Slack, Notion, Composio, MCP, ...): content the +/// user did not write, which v1 kept out of external-effect tool decisions. +pub const EXTERNAL_SYNC_TAG: &str = "taint:external_sync"; + /// One `memory_docs` row. #[derive(Debug, Clone)] struct DocRow { document_id: String, namespace: String, logical_namespace: Option, + /// Whether v1 marked the row as synced from an external service. + external: bool, title: String, content: String, tags_json: String, @@ -109,9 +122,15 @@ fn rows(ws: &LegacyWorkspace, after: Option<&str>, limit: usize) -> Result ?1) \ + updated_at, {taint} FROM memory_docs WHERE (?1 IS NULL OR document_id > ?1) \ ORDER BY document_id LIMIT ?2" ); let mut stmt = ws.memory.prepare(&sql)?; @@ -125,11 +144,24 @@ fn rows(ws: &LegacyWorkspace, after: Option<&str>, limit: usize) -> Result>(5)?.unwrap_or_default(), metadata_json: row.get::<_, Option>(6)?.unwrap_or_default(), updated_at: row.get(7)?, + external: is_external(row.get::<_, Option>(8)?.as_deref()), }) })?; Ok(rows.collect::>()?) } +/// Decodes v1's `taint` column, failing closed: only `internal` is trusted. +fn is_external(taint: Option<&str>) -> bool { + !taint.is_some_and(|value| value.trim().eq_ignore_ascii_case("internal")) +} + +/// Adds [`EXTERNAL_SYNC_TAG`] to `tags` when `row` was externally synced. +fn mark_taint(row: &DocRow, tags: &mut Vec) { + if row.external { + push_unique(tags, EXTERNAL_SYNC_TAG.to_string()); + } +} + fn logical_namespace(row: &DocRow) -> String { match &row.logical_namespace { Some(logical) if !logical.trim().is_empty() => logical.clone(), @@ -179,6 +211,7 @@ fn document(ws: &LegacyWorkspace, row: &DocRow, logical: String) -> Option) -> Option return None; } meta.tags = class.into_iter().collect(); + mark_taint(row, &mut meta.tags); meta.observed_at = updated; return Some(StoreItem::Learning { text: row.content.clone(), @@ -229,6 +263,7 @@ fn learning(ws: &LegacyWorkspace, row: &DocRow, class: Option) -> Option .as_deref() .map_or(LearningKind::Other, convert::learning_kind); meta.tags = class.into_iter().collect(); + mark_taint(row, &mut meta.tags); meta.observed_at = map .get("observed_at") .and_then(Value::as_f64) @@ -254,6 +289,7 @@ fn global(ws: &LegacyWorkspace, row: &DocRow) -> Option { } let mut meta = import_meta(ws, format!("memory_docs:{}", row.document_id)); meta.tags = vec!["global".to_string()]; + mark_taint(row, &mut meta.tags); meta.observed_at = row.updated_at.and_then(convert::from_unix_seconds); Some(StoreItem::Learning { text: row.content.clone(), diff --git a/crates/tinymemory-integrations/src/import/sections/memory_docs_tests.rs b/crates/tinymemory-integrations/src/import/sections/memory_docs_tests.rs index c1e14b28..49ae137d 100644 --- a/crates/tinymemory-integrations/src/import/sections/memory_docs_tests.rs +++ b/crates/tinymemory-integrations/src/import/sections/memory_docs_tests.rs @@ -27,3 +27,13 @@ fn classifies_logical_namespaces() { assert_eq!(classify("eventually"), RowClass::Document); assert_eq!(classify("user_notes"), RowClass::Document); } + +#[test] +fn decodes_taint_failing_closed() { + assert!(!is_external(Some("internal"))); + assert!(!is_external(Some(" INTERNAL "))); + assert!(is_external(Some("external_sync"))); + assert!(is_external(Some("sideloaded"))); + assert!(is_external(Some(""))); + assert!(is_external(None)); +} diff --git a/crates/tinymemory-integrations/src/import/sections/mod.rs b/crates/tinymemory-integrations/src/import/sections/mod.rs index 37ee230c..eb98e145 100644 --- a/crates/tinymemory-integrations/src/import/sections/mod.rs +++ b/crates/tinymemory-integrations/src/import/sections/mod.rs @@ -12,6 +12,8 @@ mod episodic; mod memory_docs; mod profile; +pub use memory_docs::EXTERNAL_SYNC_TAG; + use tinymemory_api::{MemoryMeta, SourceKind, StoreItem}; use crate::import::checkpoint::{Checkpoint, ChunkCursor}; diff --git a/crates/tinymemory-integrations/src/import/workspace/schema.rs b/crates/tinymemory-integrations/src/import/workspace/schema.rs index cc07f083..1f38ddc5 100644 --- a/crates/tinymemory-integrations/src/import/workspace/schema.rs +++ b/crates/tinymemory-integrations/src/import/workspace/schema.rs @@ -58,6 +58,8 @@ const CHUNK_COLUMNS: [&str; 7] = [ pub(crate) struct MemorySchema { /// `memory_docs.logical_namespace`. pub(crate) logical_namespace: bool, + /// `memory_docs.taint`. + pub(crate) taint: bool, /// `episodic_log.tool_calls_json`. pub(crate) tool_calls_json: bool, /// `user_profile.state`. @@ -89,6 +91,7 @@ impl MemorySchema { let profile = columns(conn, "user_profile")?; Ok(Ok(Self { logical_namespace: docs.contains("logical_namespace"), + taint: docs.contains("taint"), tool_calls_json: episodic.contains("tool_calls_json"), profile_state: profile.contains("state"), profile_user_state: profile.contains("user_state"), diff --git a/crates/tinymemory-integrations/tests/legacy_import.rs b/crates/tinymemory-integrations/tests/legacy_import.rs index 3cdc3e16..fd716a2c 100644 --- a/crates/tinymemory-integrations/tests/legacy_import.rs +++ b/crates/tinymemory-integrations/tests/legacy_import.rs @@ -13,7 +13,7 @@ use tinymemory_api::{ DocumentBody, LearningKind, Role, SourceKind, StoreItem, ToolCallRef, TurnRange, }; use tinymemory_integrations::import::{ - Checkpoint, ChunkCursor, Error, ImportedItem, LegacyWorkspace, + Checkpoint, ChunkCursor, EXTERNAL_SYNC_TAG, Error, ImportedItem, LegacyWorkspace, }; const T0: f64 = 1_700_000_000.0; @@ -645,6 +645,117 @@ fn resuming_from_any_checkpoint_yields_exactly_the_remainder() { assert_eq!(last.profile.as_deref(), Some("f2")); } +#[test] +fn tags_externally_synced_rows_in_every_memory_docs_section() { + let (dir, conn) = workspace(support::MEMORY_DDL); + let rows = [ + // (id, namespace, logical, content, taint) + ( + "e1", + "document_gmail", + "document:gmail", + "Invoice due Friday", + "external_sync", + ), + ( + "e2", + "learning_style", + "learning:style", + r#"{"class":"style","key":"tone","value":"terse"}"#, + "external_sync", + ), + ( + "e3", + "global", + "global", + "Always cc finance", + "external_sync", + ), + ( + "e4", + "document_web", + "document:web", + "Unknown taint", + "sideloaded", + ), + ("e5", "document_web", "document:web", "Blank taint", ""), + ( + "i1", + "document_notes", + "document:notes", + "My own note", + "internal", + ), + ( + "i2", + "document_notes", + "document:notes", + "Spelled loosely", + " Internal ", + ), + ]; + for (id, namespace, logical, content, taint) in rows { + doc( + &conn, + id, + namespace, + Some(logical), + "", + content, + "[]", + "{}", + T0, + ); + conn.execute( + "UPDATE memory_docs SET taint = ?2 WHERE document_id = ?1", + rusqlite::params![id, taint], + ) + .unwrap(); + } + let ws = LegacyWorkspace::open(dir.path()).unwrap(); + let items = all(&ws); + let tags = |id: &str| { + find(&items, &format!("memory_docs:{id}")) + .meta() + .tags + .clone() + }; + + assert_eq!(tags("e1"), ["ns:document:gmail", EXTERNAL_SYNC_TAG]); + assert!(matches!( + find(&items, "memory_docs:e2"), + StoreItem::Learning { .. } + )); + assert_eq!(tags("e2"), ["style", EXTERNAL_SYNC_TAG]); + assert_eq!(tags("e3"), ["global", EXTERNAL_SYNC_TAG]); + // v1 decodes anything but `internal` as external; so does the importer. + assert!(tags("e4").contains(&EXTERNAL_SYNC_TAG.to_string())); + assert!(tags("e5").contains(&EXTERNAL_SYNC_TAG.to_string())); + assert_eq!(tags("i1"), ["ns:document:notes"]); + assert_eq!(tags("i2"), ["ns:document:notes"]); +} + +#[test] +fn a_store_without_the_taint_column_reads_as_internal() { + let (dir, conn) = workspace(OLD_MEMORY_DDL); + conn.execute_batch( + "INSERT INTO memory_docs (document_id, namespace, key, title, content, source_type, + priority, tags_json, metadata_json, category, created_at, updated_at, markdown_rel_path) + VALUES ('a', 'document_old', 'k', 'Old', 'old body', 'doc', 'n', '[]', '{}', 'core', 1, 1, '');", + ) + .unwrap(); + let ws = LegacyWorkspace::open(dir.path()).unwrap(); + let items = all(&ws); + assert!(items.iter().all(|imported| { + !imported + .item + .meta() + .tags + .iter() + .any(|t| t == EXTERNAL_SYNC_TAG) + })); +} + #[test] fn imports_an_early_v1_store_without_optional_columns() { let (dir, conn) = workspace(OLD_MEMORY_DDL);