diff --git a/Sensor/README.md b/Sensor/README.md index 89cbbd2..1b47d69 100644 --- a/Sensor/README.md +++ b/Sensor/README.md @@ -112,6 +112,9 @@ are read: - **SQLite** (current releases) — `opencode.db`, or `opencode-.db` on non-stable channels. Opened read-only so a running opencode process is never disturbed. + A database is used only if it contains the `session`, `message` and `part` tables; + otherwise the other candidate databases and then the JSON tree are tried, and a + database without those tables is reported as `unsupported_schema`. - **JSON file tree** (older releases) — a `storage/` directory of per-session, per-message and per-part JSON files, in both the project-scoped and legacy layouts. diff --git a/Sensor/adr_sensor/parsers/opencode_parser.py b/Sensor/adr_sensor/parsers/opencode_parser.py index 8223c67..c20b59f 100644 --- a/Sensor/adr_sensor/parsers/opencode_parser.py +++ b/Sensor/adr_sensor/parsers/opencode_parser.py @@ -17,7 +17,9 @@ conversation record for local CLI usage; the newer ``session_message`` / ``event`` tables are an additional event-sourced projection used for other purposes and were empty in every real session captured while building this - parser, so they are not read here. + parser, so they are not read here. A database is only used if it has the + ``session`` / ``message`` / ``part`` tables; otherwise the remaining + candidate databases and then the JSON tree below are tried. 2. JSON file tree (older, pre-SQLite versions): a ``storage/`` directory holding one JSON file per session / message / part. Both the newer project-scoped @@ -79,10 +81,16 @@ } ) +# Tables a SQLite database must contain to be read as opencode session history. +REQUIRED_SQLITE_TABLES = frozenset({"session", "message", "part"}) + class OpencodeParser(BaseParser): """Parser for opencode session logs (SQLite and legacy JSON backends).""" + # Set by _detect_backend when databases exist but none has the session tables. + _schema_mismatch = False + def __init__(self, max_age_days: int = MAX_LOG_AGE_DAYS): self.max_age_days = max_age_days self.base_dir, self.backend, self.db_path = self._detect_backend() @@ -112,38 +120,71 @@ def _candidate_base_dirs() -> List[Path]: return unique @staticmethod - def _find_db_file(base: Path) -> Optional[Path]: - """Locate the opencode SQLite database under a candidate data directory. + def _find_db_files(base: Path) -> List[Path]: + """List candidate opencode SQLite databases under a data directory. The ``OPENCODE_DB`` environment variable can override the filename (or be an absolute path / ``:memory:``); otherwise the file is ``opencode.db`` for stable channels or ``opencode-.db`` for anything else (e.g. a nightly build), so we fall back to globbing for that pattern too. + Candidates are returned in that priority order, without duplicates. """ + candidates: List[Path] = [] + env_db = os.environ.get("OPENCODE_DB") if env_db and env_db != ":memory:": candidate = Path(env_db) if not candidate.is_absolute(): candidate = base / env_db if candidate.exists(): - return candidate + candidates.append(candidate) default = base / "opencode.db" - if default.exists(): - return default + if default.exists() and default not in candidates: + candidates.append(default) if base.exists(): for candidate in sorted(base.glob("opencode-*.db")): - if candidate.exists(): - return candidate - return None + if candidate.exists() and candidate not in candidates: + candidates.append(candidate) + return candidates + + @staticmethod + def _find_db_file(base: Path) -> Optional[Path]: + """Return the highest-priority candidate database, if any.""" + candidates = OpencodeParser._find_db_files(base) + return candidates[0] if candidates else None + + @staticmethod + def _has_session_schema(db_path: Path) -> bool: + """Return True if the database has opencode's session tables.""" + conn = None + try: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + rows = conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall() + except (sqlite3.Error, OSError): + return False + finally: + if conn is not None: + conn.close() + return REQUIRED_SQLITE_TABLES <= {row[0] for row in rows} def _detect_backend(self) -> Tuple[Path, Optional[str], Optional[Path]]: - """Return (base_dir, backend, db_path) where backend is 'sqlite', 'json' or None.""" + """Return (base_dir, backend, db_path) where backend is 'sqlite', 'json' or None. + + A database is selected only if it contains the session tables; otherwise + the remaining candidates and then the JSON ``storage/`` tree are tried. + This runs before per-run diagnostics are reset, so a database without + the expected tables is only remembered here and reported by parse_all. + """ + self._schema_mismatch = False for base in self._candidate_base_dirs(): - db_path = self._find_db_file(base) - if db_path is not None: - return base, "sqlite", db_path + db_files = self._find_db_files(base) + for db_path in db_files: + if self._has_session_schema(db_path): + return base, "sqlite", db_path + if db_files: + self._schema_mismatch = True storage_dir = base / "storage" if storage_dir.exists(): return base, "json", None @@ -160,9 +201,16 @@ def parse_all(self) -> List[AgentEvent]: if self.backend == "json": storage_dir = self.base_dir / "storage" + if self._schema_mismatch: + logger.info("[OPENCODE] SQLite database lacks the session tables; reading JSON storage") logger.info("[OPENCODE] Reading JSON logs from %s", storage_dir) return self._parse_json_storage(storage_dir) + if self._schema_mismatch: + logger.info("[OPENCODE] Database found but it does not contain the expected session tables") + self.record_diagnostic("unsupported_schema") + return [] + logger.info("[OPENCODE] No logs found at %s", self.base_dir) self.record_diagnostic("input_missing") return [] @@ -193,9 +241,23 @@ def _parse_sqlite(self, db_path: Path) -> List[AgentEvent]: ) if entry and entry.has_meaningful_content(): entries.append(entry) + except sqlite3.OperationalError as e: + # A missing table affects every session: stop and report it once. + if "no such table" in str(e): + raise + self.record_diagnostic("session_build_error") + print(f"[OPENCODE] Error processing session {session_id}: {e}") except Exception as e: self.record_diagnostic("session_build_error") logger.warning("[OPENCODE] Error processing session %s: %s", session_id, e) + except sqlite3.OperationalError as e: + # The schema can change between backend detection and parsing. + if "no such table" in str(e): + self.record_diagnostic("unsupported_schema") + logger.warning("[OPENCODE] Expected session tables not found in database") + else: + self.record_diagnostic("database_error") + logger.warning("[OPENCODE] Error reading database: %s", e) except Exception as e: self.record_diagnostic("database_error") logger.error("[OPENCODE] Error reading database: %s", e) diff --git a/Sensor/tests/test_parser_diagnostics.py b/Sensor/tests/test_parser_diagnostics.py index 7c79a0e..6500549 100644 --- a/Sensor/tests/test_parser_diagnostics.py +++ b/Sensor/tests/test_parser_diagnostics.py @@ -151,7 +151,7 @@ def test_diagnostics_survive_when_every_record_is_rejected(tmp_path, parser_clas assert parser.get_diagnostics()["record_shape_error"] >= 1 -@pytest.mark.parametrize("parser_class", [CursorParser, OpencodeParser, WarpParser]) +@pytest.mark.parametrize("parser_class", [CursorParser, WarpParser]) def test_incompatible_database_reports_failure(tmp_path, parser_class): parser = isolated_parser(parser_class, tmp_path) parser.db_path = tmp_path / "synthetic.db" @@ -162,6 +162,16 @@ def test_incompatible_database_reports_failure(tmp_path, parser_class): assert parser.get_diagnostics()["database_error"] >= 1 +def test_opencode_database_without_tables_reports_schema_drift(tmp_path): + parser = isolated_parser(OpencodeParser, tmp_path) + parser.db_path = tmp_path / "synthetic.db" + parser.backend = "sqlite" + with sqlite3.connect(parser.db_path): + pass + assert parser.parse_all() == [] + assert parser.get_diagnostics() == {"unsupported_schema": 1} + + def test_dsh_new_generation_reports_schema_drift_without_reading_payload(tmp_path): parser = isolated_parser(DshParser, tmp_path) parser.base_path = tmp_path diff --git a/Sensor/tests/test_parsers.py b/Sensor/tests/test_parsers.py index 2db5dbc..244d764 100644 --- a/Sensor/tests/test_parsers.py +++ b/Sensor/tests/test_parsers.py @@ -2119,6 +2119,14 @@ def _build_opencode_db(db_path, sessions, messages, parts): conn.close() +def _build_non_opencode_db(db_path): + """Create a SQLite database that lacks opencode's session tables.""" + conn = sqlite3.connect(str(db_path)) + conn.execute("CREATE TABLE migrations (id INTEGER PRIMARY KEY, name TEXT)") + conn.commit() + conn.close() + + def _make_opencode_parser(base_dir, max_age_days=0): """Build an OpencodeParser whose backend detection points at base_dir.""" with patch.object(OpencodeParser, "_candidate_base_dirs", return_value=[base_dir]): @@ -2132,7 +2140,7 @@ def test_no_backend_found(self, tmp_path): assert parser.parse_all() == [] def test_sqlite_backend_detected(self, tmp_path): - (tmp_path / "opencode.db").touch() + _build_opencode_db(tmp_path / "opencode.db", [], [], []) parser = _make_opencode_parser(tmp_path) assert parser.backend == "sqlite" @@ -2142,37 +2150,102 @@ def test_json_backend_detected(self, tmp_path): assert parser.backend == "json" def test_sqlite_takes_priority_over_json(self, tmp_path): - (tmp_path / "opencode.db").touch() + _build_opencode_db(tmp_path / "opencode.db", [], [], []) (tmp_path / "storage").mkdir() parser = _make_opencode_parser(tmp_path) assert parser.backend == "sqlite" def test_channel_suffixed_db_detected(self, tmp_path): """Non-stable channels write opencode-.db instead.""" - (tmp_path / "opencode-nightly.db").touch() + _build_opencode_db(tmp_path / "opencode-nightly.db", [], [], []) parser = _make_opencode_parser(tmp_path) assert parser.backend == "sqlite" assert parser.db_path.name == "opencode-nightly.db" def test_default_db_preferred_over_channel_suffixed(self, tmp_path): - (tmp_path / "opencode.db").touch() - (tmp_path / "opencode-dev.db").touch() + _build_opencode_db(tmp_path / "opencode.db", [], [], []) + _build_opencode_db(tmp_path / "opencode-dev.db", [], [], []) parser = _make_opencode_parser(tmp_path) assert parser.db_path.name == "opencode.db" def test_opencode_db_env_override(self, tmp_path, monkeypatch): custom = tmp_path / "custom.db" - custom.touch() + _build_opencode_db(custom, [], [], []) monkeypatch.setenv("OPENCODE_DB", str(custom)) parser = _make_opencode_parser(tmp_path) assert parser.db_path == custom def test_opencode_db_memory_value_ignored(self, tmp_path, monkeypatch): monkeypatch.setenv("OPENCODE_DB", ":memory:") + _build_opencode_db(tmp_path / "opencode.db", [], [], []) + parser = _make_opencode_parser(tmp_path) + assert parser.db_path.name == "opencode.db" + + def test_channel_db_selected_when_default_lacks_tables(self, tmp_path): + """An empty or migration-only opencode.db must not shadow a populated channel DB.""" + _build_non_opencode_db(tmp_path / "opencode.db") + _build_opencode_db(tmp_path / "opencode-nightly.db", [], [], []) + parser = _make_opencode_parser(tmp_path) + assert parser.backend == "sqlite" + assert parser.db_path.name == "opencode-nightly.db" + + def test_db_without_session_tables_falls_back_to_json(self, tmp_path): + _build_non_opencode_db(tmp_path / "opencode.db") + (tmp_path / "storage").mkdir() + parser = _make_opencode_parser(tmp_path) + assert parser.backend == "json" + assert parser.db_path is None + parser.reset_diagnostics() + parser.parse_all() + assert "unsupported_schema" not in parser.get_diagnostics() + + def test_db_without_session_tables_and_no_json_reports_unsupported_schema(self, tmp_path): + _build_non_opencode_db(tmp_path / "opencode.db") + parser = _make_opencode_parser(tmp_path) + assert parser.backend is None + parser.reset_diagnostics() + assert parser.parse_all() == [] + assert parser.get_diagnostics() == {"unsupported_schema": 1} + + def test_empty_db_file_is_not_selected(self, tmp_path): (tmp_path / "opencode.db").touch() parser = _make_opencode_parser(tmp_path) + assert parser.backend is None + + def test_env_override_without_tables_falls_through_to_default(self, tmp_path, monkeypatch): + _build_non_opencode_db(tmp_path / "custom.db") + _build_opencode_db(tmp_path / "opencode.db", [], [], []) + monkeypatch.setenv("OPENCODE_DB", "custom.db") + parser = _make_opencode_parser(tmp_path) assert parser.db_path.name == "opencode.db" + def test_unreadable_candidate_is_skipped(self, tmp_path): + _build_opencode_db(tmp_path / "opencode.db", [], [], []) + _build_opencode_db(tmp_path / "opencode-dev.db", [], [], []) + real_connect = sqlite3.connect + + def flaky_connect(target, *args, **kwargs): + if "opencode.db" in str(target): + raise sqlite3.OperationalError("unable to open database file") + return real_connect(target, *args, **kwargs) + + with patch("adr_sensor.parsers.opencode_parser.sqlite3.connect", side_effect=flaky_connect): + parser = _make_opencode_parser(tmp_path) + assert parser.db_path.name == "opencode-dev.db" + + def test_find_db_files_lists_candidates_in_priority_order(self, tmp_path, monkeypatch): + for name in ("opencode-nightly.db", "opencode-beta.db", "opencode.db", "custom.db"): + (tmp_path / name).touch() + monkeypatch.setenv("OPENCODE_DB", "custom.db") + names = [p.name for p in OpencodeParser._find_db_files(tmp_path)] + assert names == ["custom.db", "opencode.db", "opencode-beta.db", "opencode-nightly.db"] + assert OpencodeParser._find_db_file(tmp_path).name == "custom.db" + + def test_find_db_files_does_not_repeat_env_override(self, tmp_path, monkeypatch): + (tmp_path / "opencode.db").touch() + monkeypatch.setenv("OPENCODE_DB", "opencode.db") + assert OpencodeParser._find_db_files(tmp_path) == [tmp_path / "opencode.db"] + def test_xdg_data_home_is_first_candidate(self, tmp_path, monkeypatch): monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path)) assert OpencodeParser._candidate_base_dirs()[0] == tmp_path / "opencode" @@ -2182,6 +2255,28 @@ def test_default_max_age_days(self): class TestOpencodeParserSqlite: + def test_missing_table_during_parse_records_unsupported_schema(self, tmp_path): + db_path = tmp_path / "opencode.db" + conn = sqlite3.connect(str(db_path)) + conn.execute("CREATE TABLE session (id TEXT PRIMARY KEY, time_updated INTEGER)") + conn.execute("INSERT INTO session VALUES ('ses_1', ?)", (NOW_MS,)) + conn.commit() + conn.close() + parser = _make_opencode_parser(tmp_path) + parser.reset_diagnostics() + assert parser._parse_sqlite(db_path) == [] + diagnostics = parser.get_diagnostics() + assert diagnostics.get("unsupported_schema") == 1 + assert "database_error" not in diagnostics + + def test_unreadable_database_still_records_database_error(self, tmp_path): + db_path = tmp_path / "opencode.db" + db_path.write_bytes(b"not a sqlite database" * 10) + parser = _make_opencode_parser(tmp_path) + parser.reset_diagnostics() + assert parser._parse_sqlite(db_path) == [] + assert parser.get_diagnostics() == {"database_error": 1} + def test_parses_conversation_with_tool(self, tmp_path): db_path = tmp_path / "opencode.db" _build_opencode_db(