Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Sensor/CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,8 @@ logger.warning("[MY_AGENT] Skipped unreadable file", extra={"phase": "parse"})
- Use `WARNING` for recoverable problems (a skipped file or record) and `ERROR`
when a whole source or output step fails. Keep recording fixed diagnostic codes
with `record_diagnostic()`; log messages do not replace them.
- Never log prompts, tool arguments or results, or other captured content.
- Never log prompts, tool arguments or results, or other captured content. With
`--log-file`, messages are persisted to `sensor_runtime_*.jsonl`.
- Explicit report output, such as `AgentObserver.display_summary()`, stays as `print()`.

## Testing Guidelines
Expand Down
23 changes: 23 additions & 0 deletions Sensor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,29 @@ session IDs, exception messages, or tracebacks. This is a separate operational
schema, **not redaction of captured telemetry**. Legacy console previews/errors and
older entries already present in `error.log` are not sanitized by this change.

### Runtime log files

`--log-file` also writes the run's log messages as JSON lines next to the
diagnostics: `sensor_runtime_errors.jsonl` holds warnings and errors, and
`sensor_runtime_debug.jsonl` holds debug and info messages whatever the console
`--log-level` is. Each file rotates at 1 MiB with two backups. Runtime logging is
off by default. If the files cannot be opened, the sensor prints one warning and
keeps printing warnings and errors to stderr.

Each record has `timestamp`, `level`, `component`, `function`, `phase`,
`sensor_version`, `exception_type`, `message` and, for errors with a traceback,
`stack`:

```json
{"timestamp":"2026-01-01T12:00:00.000+00:00","level":"WARNING","component":"parsers.cline_parser","function":"parse_all","phase":null,"sensor_version":"0.1.0","exception_type":"KeyError","message":"[CLINE] Error parsing task /home/me/.cline/tasks/123: 'ts'"}
```

**Privacy:** unlike the diagnostics above, `message` and `stack` can contain local
file paths and error text. Log call sites do not log prompts or tool content, but
treat these files like other local logs. `--log-file-content-free` drops `message`
and `stack` and keeps the remaining fields. `--log-identity` adds `username` and
`hostname` to each record; it is opt-in because it identifies the machine and user.

## Output Schema

### AgentEvent
Expand Down
27 changes: 26 additions & 1 deletion Sensor/adr_sensor/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from .exporters.delivery_checkpoint import DeliveryCheckpoint, DeliveryCheckpointError
from .exporters.opentelemetry import OpenTelemetryExportError, OpenTelemetryLogExporter
from .observer import AgentObserver
from .sensor_log import append_rotating_line, set_console_level
from .sensor_log import append_rotating_line, disable_runtime_log, enable_runtime_log, set_console_level

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -125,9 +125,33 @@ def main():
verbosity.add_argument(
"-q", "--quiet", action="store_true", help="Print only warnings and errors (same as --log-level warning)"
)
parser.add_argument(
"--log-file",
action="store_true",
help="Also write runtime logs as rotating JSON lines (sensor_runtime_*.jsonl) in the output directory",
)
parser.add_argument(
"--log-file-content-free",
action="store_true",
help="Omit messages and tracebacks from --log-file records (no paths or error text)",
)
parser.add_argument(
"--log-identity",
action="store_true",
help="Add the local username and hostname to --log-file records",
)

args = parser.parse_args()
for option in ("log_file_content_free", "log_identity"):
if getattr(args, option) and not args.log_file:
parser.error(f"--{option.replace('_', '-')} requires --log-file")
set_console_level("warning" if args.quiet else args.log_level)
if args.log_file:
enable_runtime_log(
args.output_dir or Path.cwd() / "output",
include_details=not args.log_file_content_free,
include_identity=args.log_identity,
)

otel_config = None
if args.otel_config is not None:
Expand Down Expand Up @@ -322,6 +346,7 @@ def _delta(attr: str) -> float:
append_rotating_line(log_path, json.dumps(record, separators=(",", ":"), ensure_ascii=False))
except Exception:
pass
disable_runtime_log()


if __name__ == "__main__":
Expand Down
126 changes: 126 additions & 0 deletions Sensor/adr_sensor/sensor_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,20 @@
``diagnostics.py``: they describe what the sensor did, not how healthy a run was.
"""

import getpass
import json
import logging
import socket
import sys
from datetime import datetime, timezone
from logging.handlers import RotatingFileHandler
from pathlib import Path

from . import __version__

LOGGER_NAME = "adr_sensor"
RUNTIME_ERRORS_LOG = "sensor_runtime_errors.jsonl"
RUNTIME_DEBUG_LOG = "sensor_runtime_debug.jsonl"

# Size-based rotation shared by the sensor's append-only files.
MAX_LOG_BYTES = 1024 * 1024
Expand Down Expand Up @@ -68,6 +76,65 @@ def component_for(record: logging.LogRecord) -> str:
return name[len(prefix) :] if name.startswith(prefix) else name


def _exception_type(record: logging.LogRecord):
"""Name the exception attached to *record*, or the first one passed as an argument."""
if record.exc_info and record.exc_info[0] is not None:
return record.exc_info[0].__name__
args = record.args if isinstance(record.args, tuple) else ()
for arg in args:
if isinstance(arg, BaseException):
return type(arg).__name__
return None


def _local_identity():
"""Return the local ``(username, hostname)``, with ``None`` for unknown parts."""
try:
username = getpass.getuser()
except Exception:
username = None
try:
hostname = socket.gethostname() or None
except OSError:
hostname = None
return username, hostname


class JsonRecordFormatter(logging.Formatter):
"""Format records as one JSON object per line for the runtime log files.

Every record carries ``timestamp``, ``level``, ``component``, ``function``,
``phase``, ``sensor_version`` and ``exception_type``. The rendered
``message`` and, when an exception is attached, its ``stack`` are included
unless *include_details* is false, which keeps records free of paths and
error text. *include_identity* adds the local ``username`` and ``hostname``,
looked up once.
"""

def __init__(self, *, include_details: bool = True, include_identity: bool = False) -> None:
super().__init__()
self.include_details = include_details
self.identity = _local_identity() if include_identity else None

def format(self, record: logging.LogRecord) -> str:
data = {
"timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(timespec="milliseconds"),
"level": record.levelname,
"component": component_for(record),
"function": record.funcName,
"phase": getattr(record, "phase", None),
"sensor_version": __version__,
"exception_type": _exception_type(record),
}
if self.include_details:
data["message"] = record.getMessage()
if record.exc_info:
data["stack"] = self.formatException(record.exc_info)
if self.identity is not None:
data["username"], data["hostname"] = self.identity
return json.dumps(data, ensure_ascii=False, separators=(",", ":"), default=str)


def _console_handlers():
return [handler for handler in logger.handlers if isinstance(handler, _StdStreamHandler)]

Expand Down Expand Up @@ -96,6 +163,65 @@ def set_console_level(level) -> None:
handler.setLevel(level)


class _RuntimeFileHandler(RotatingFileHandler):
"""Rotating runtime log file that never interrupts or floods a run."""

def handleError(self, record: logging.LogRecord) -> None:
if getattr(self, "_reported_error", False):
return
self._reported_error = True
sys.stderr.write(f"[ADR] Unable to write the runtime log {Path(self.baseFilename).name}; continuing.\n")


def _runtime_handlers():
return [handler for handler in logger.handlers if isinstance(handler, _RuntimeFileHandler)]


def disable_runtime_log() -> None:
"""Detach and close the runtime log files, if any."""
for handler in _runtime_handlers():
logger.removeHandler(handler)
try:
handler.close()
except (OSError, ValueError):
pass


def enable_runtime_log(directory: Path, *, include_details: bool = True, include_identity: bool = False) -> bool:
"""Also write sensor log records to rotating JSON-lines files in *directory*.

WARNING and above go to ``sensor_runtime_errors.jsonl``; DEBUG and INFO go
to ``sensor_runtime_debug.jsonl`` whatever the console level is. Each file
rotates at 1 MiB with two backups. Calling it again replaces the previous
files. If the files cannot be opened, warnings and errors still reach
stderr through the console handlers and this returns ``False``.
"""
disable_runtime_log()
formatter = JsonRecordFormatter(include_details=include_details, include_identity=include_identity)
handlers = []
try:
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
for name, level_filter in (
(RUNTIME_ERRORS_LOG, _WarningAndAboveFilter()),
(RUNTIME_DEBUG_LOG, _BelowWarningFilter()),
):
handler = _RuntimeFileHandler(
directory / name, maxBytes=MAX_LOG_BYTES, backupCount=LOG_BACKUP_COUNT, encoding="utf-8"
)
handler.addFilter(level_filter)
handler.setFormatter(formatter)
handlers.append(handler)
except OSError:
for handler in handlers:
handler.close()
logger.warning("[ADR] Unable to open the runtime log; warnings and errors are printed to stderr only.")
return False
for handler in handlers:
logger.addHandler(handler)
return True


def append_rotating_line(
path: Path, line: str, *, max_bytes: int = MAX_LOG_BYTES, backup_count: int = LOG_BACKUP_COUNT
) -> None:
Expand Down
Loading
Loading