diff --git a/apps/api/src/routes/internal/ai-sessions.http.test.ts b/apps/api/src/routes/internal/ai-sessions.http.test.ts index 7b1c49baa..b44b86f76 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.test.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.test.ts @@ -2,11 +2,13 @@ import { describe, expect, it } from "@effect/vitest" import { AiSessionsInternalApiGroup, + AI_OVERVIEW_BREAKDOWN_MAX, AI_SESSION_SPANS_MAX_SPANS, AI_SESSION_SUMMARY_MAX_TURNS, CurrentTenant, V1SchemaErrors, V1UnexpectedErrors, + WarehouseQueryError, } from "@maple/domain/http" import { WarehouseResponseLimitError } from "@maple/query-engine/execution" @@ -1094,3 +1096,447 @@ describe("POST /internal/ai-sessions/summary", () => { } }) }) + +/** + * The overview's two reads. What matters here is the composition the route + * does and not the SQL: the previous window is computed server-side, the + * summary is two reads folded into one response, and the breakdown pairs each + * key's two periods and zero-fills the one that has no row. + */ +const OVERVIEW_MEASURES = { + sessions: "4", + erroredSessions: "1", + llmCalls: 9, + // The failures' own population: the model-call SPANS, mirrors included, so + // the rate the client takes cannot pass 100%. + llmCallSpans: "11", + erroredLlmCalls: "2", + toolCalls: "6", + erroredToolCalls: "1", + cost: 0.42, + pricedLlmCalls: 7, + tokens: 18_400, + inputTokens: 12_000, + cacheReadTokens: 4_000, + cacheWriteTokens: 0, + outputTokens: 2_000, + reasoningTokens: 400, + sessionDurationP50Ns: 600_000_000, + sessionDurationP95Ns: 900_000_000, +} + +/** One row of the totals or series union, in the wire shape it decodes from. */ +const overviewRow = (overrides: Record) => ({ ...OVERVIEW_MEASURES, ...overrides }) + +describe("POST /internal/ai-sessions/overview/summary", () => { + const SUMMARY_BODY = { ...WINDOW, bucketSeconds: 300 } + + const TOTALS = [ + overviewRow({ period: "current" }), + overviewRow({ period: "previous", sessions: "2", cost: 0.2 }), + ] + const SERIES = [ + overviewRow({ period: "current", bucket: "2026-08-19T09:00:00.000Z", sessions: "1" }), + overviewRow({ period: "current", bucket: "2026-08-19T10:00:00.000Z", sessions: "3" }), + overviewRow({ period: "previous", bucket: "2026-08-19T07:00:00.000Z", sessions: "2" }), + ] + + const summaryHarness = () => { + const contexts: Array = [] + const sqlByContext = new Map() + const harness = makeHarness({ + compiledQuery: (_tenant, compiled, options) => { + const context = options?.context ?? "" + contexts.push(context) + sqlByContext.set(context, compiledQueryOf(compiled).sql) + return compiledQueryOf(compiled) + .decodeRows(context === "aiOverviewTotals" ? TOTALS : SERIES) + .pipe(Effect.orDie) + }, + }) + return { harness, contexts, sqlByContext } + } + + it("answers from two index reads, the tiles beside the chart", async () => { + const { harness, contexts, sqlByContext } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + // Two groupings of one population: quantiles do not merge, so the + // tiles cannot be folded from the chart. + expect([...contexts].sort()).toEqual(["aiOverviewSeries", "aiOverviewTotals"]) + for (const sql of sqlByContext.values()) { + expect(sql).toContain("FROM ai_trace_index") + expect(sql).not.toContain("trace_detail_spans") + expect(sql).not.toContain("__PARAM_") + } + // The chart's bucket reaches the read as the interval it was asked for. + expect(sqlByContext.get("aiOverviewSeries")).toContain("INTERVAL 300 SECOND") + expect(sqlByContext.get("aiOverviewTotals")).not.toContain("INTERVAL") + } finally { + await harness.dispose() + } + }) + + it("compares against the window of equal length ending where the caller's begins", async () => { + const { harness, sqlByContext } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + // The caller asked for 09:00–11:00, so the comparison is 07:00–09:00 — + // computed here, never asked for, so a delta cannot be taken against a + // window of a different size. + const sql = sqlByContext.get("aiOverviewTotals") ?? "" + expect(sql).toContain("Timestamp >= '2026-08-19 07:00:00'") + // `[07:00, 09:00)`: the comparison ends where the caller's window + // begins, so 09:00:00 itself is measured in one window and not in two. + expect(sql).toContain("Timestamp < '2026-08-19 09:00:00'") + expect(sql).not.toContain("Timestamp <= '2026-08-19 09:00:00'") + expect(sql).toContain(`Timestamp >= '${WINDOW.startTime}'`) + expect(sql).toContain(`Timestamp <= '${WINDOW.endTime}'`) + } finally { + await harness.dispose() + } + }) + + it("folds the two reads into the window, its comparison, and the two series", async () => { + const { harness } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + expect(response.body).toMatchObject({ + bucketSeconds: 300, + current: { sessions: 4, erroredSessions: 1, llmCalls: 9, cost: 0.42, tokens: 18_400 }, + previous: { sessions: 2, cost: 0.2 }, + }) + const series = response.body.series as ReadonlyArray> + expect(series.map((point) => point.bucket)).toEqual([ + "2026-08-19T09:00:00.000Z", + "2026-08-19T10:00:00.000Z", + ]) + // A session is filed under the bucket it started in, so the buckets sum + // to the tile. + expect(series.reduce((total, point) => total + Number(point.sessions), 0)).toBe(4) + const previousSeries = response.body.previousSeries as ReadonlyArray> + expect(previousSeries.map((point) => point.bucket)).toEqual(["2026-08-19T07:00:00.000Z"]) + } finally { + await harness.dispose() + } + }) + + it("answers an empty window with zeros rather than a missing period", async () => { + const harness = makeHarness({ + compiledQuery: (_tenant, compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), + }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + expect(response.status).toBe(200) + expect(response.body).toMatchObject({ + current: { sessions: 0, cost: 0, sessionDurationP95Ns: 0 }, + previous: { sessions: 0 }, + series: [], + previousSeries: [], + }) + } finally { + await harness.dispose() + } + }) + + it("refuses a fractional bucket with a 400 rather than a 500", async () => { + const harness = makeHarness({ + compiledQuery: () => Effect.die("the read must never run"), + }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", { + ...WINDOW, + bucketSeconds: 1.5, + }) + // `param.int` rejects a fraction inside the builder, which would be a + // 500 — the contract catches it at the boundary instead. + expect(response.status).toBe(400) + } finally { + await harness.dispose() + } + }) + + // Every filter is a payload field the handler has to hand to the builder by + // name, exactly as the list route's own case asserts it: a field the schema + // accepts and the handler forgets is a 200 that quietly ignores the toolbar + // and leaves the tiles describing a different population than the table. + it("hands the board's selection to both of the summary's reads", async () => { + const { harness, sqlByContext } = summaryHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", { + ...SUMMARY_BODY, + vendorIds: ["eve"], + models: ["claude-sonnet-5"], + hasErrors: true, + }) + expect(response.status).toBe(200) + for (const sql of sqlByContext.values()) { + expect(sql).toContain("countIf(VendorId IN ('eve')) > 0") + expect(sql).toContain("countIf(Model IN ('claude-sonnet-5')) > 0") + // The failed-session test is a session-level one, applied over the + // trace rollup — the same rule the list matches `hasErrors` by. + expect(sql).toContain("HAVING sum(errorSpans) > 0") + } + } finally { + await harness.dispose() + } + }) + + it("answers a failed warehouse read with the route's warehouse envelope", async () => { + const harness = makeHarness({ + compiledQuery: () => + Effect.fail( + new WarehouseQueryError({ + message: "Code: 241. DB::Exception: Memory limit exceeded", + pipeName: "aiOverviewTotals", + }), + ), + }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/summary", SUMMARY_BODY) + // The group declares `warehouseReadHttpErrors`, so a driver failure + // leaves the route as its own tagged error at its own status — not as + // an unexpected-error 500, and not as an empty 200. + expect(response.status).toBe(502) + expect(response.body._tag).toBe("@maple/http/errors/WarehouseQueryError") + expect(response.body.pipeName).toBe("aiOverviewTotals") + } finally { + await harness.dispose() + } + }) + + it("refuses an inverted window with a 400 rather than reading one backwards", async () => { + const harness = makeHarness({ compiledQuery: () => Effect.die("the read must never run") }) + + try { + // It reaches the partition predicate as given and answers empty, which + // reads as "nothing ran" rather than as the bad request it is. + const response = await harness.post("/internal/ai-sessions/overview/summary", { + ...SUMMARY_BODY, + startTime: WINDOW.endTime, + endTime: WINDOW.startTime, + }) + expect(response.status).toBe(400) + } finally { + await harness.dispose() + } + }) + + it("refuses a datetime the calendar does not have with a 400 rather than a 500", async () => { + const harness = makeHarness({ compiledQuery: () => Effect.die("the read must never run") }) + + try { + // The window's pattern admits it; `Date.parse` does not. It would reach + // the comparison window as a NaN and leave `toISOString` throwing. + const summary = await harness.post("/internal/ai-sessions/overview/summary", { + ...WINDOW, + startTime: "2026-13-45 99:99:99", + bucketSeconds: 300, + }) + expect(summary.status).toBe(400) + const breakdown = await harness.post("/internal/ai-sessions/overview/breakdown", { + ...WINDOW, + startTime: "2026-13-45 99:99:99", + dimension: "model", + }) + expect(breakdown.status).toBe(400) + const modelMix = await harness.post("/internal/ai-sessions/overview/model-mix", { + ...WINDOW, + startTime: "2026-13-45 99:99:99", + bucketSeconds: 300, + }) + expect(modelMix.status).toBe(400) + } finally { + await harness.dispose() + } + }) +}) + +describe("POST /internal/ai-sessions/overview/breakdown", () => { + const BREAKDOWN_BODY = { ...WINDOW, dimension: "model" } + + const ROWS = [ + overviewRow({ period: "current", key: "gpt-5.5", keyCount: 0, sessions: "3", cost: 0.9 }), + overviewRow({ period: "current", key: "claude-sonnet-5", keyCount: 0, sessions: "7", cost: 0.3 }), + overviewRow({ period: "current", key: "", keyCount: 0, sessions: "3", cost: 0.1 }), + overviewRow({ period: "previous", key: "gpt-5.5", keyCount: 0, sessions: "2", cost: 0.5 }), + overviewRow({ period: "keys", key: "", keyCount: 9, sessions: "0" }), + ] + + const breakdownHarness = (rows: ReadonlyArray> = ROWS) => { + let sql: string | undefined + const contexts: Array = [] + const harness = makeHarness({ + compiledQuery: (_tenant, compiled, options) => { + contexts.push(options?.context) + sql = compiledQueryOf(compiled).sql + return compiledQueryOf(compiled).decodeRows(rows).pipe(Effect.orDie) + }, + }) + return { harness, contexts, readSql: () => sql ?? "" } + } + + it("reads both windows and the window's key count in one query", async () => { + const { harness, contexts, readSql } = breakdownHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/breakdown", BREAKDOWN_BODY) + expect(response.status).toBe(200) + expect(contexts).toEqual(["aiOverviewBreakdown"]) + // The dimension picks the column, and a model is read over model calls. + expect(readSql()).toContain("toString(ai_trace_index.Model) AS key") + expect(readSql()).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(response.body).toMatchObject({ dimension: "model", totalKeys: 9 }) + } finally { + await harness.dispose() + } + }) + + it("ranks the keys by sessions, pairs the two periods, and keeps the unattributed row", async () => { + const { harness } = breakdownHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/breakdown", BREAKDOWN_BODY) + expect(response.status).toBe(200) + const rows = response.body.rows as ReadonlyArray> + // Busiest first, cost breaking the tie; `''` is a real key the page + // renders as unattributed rather than a gap. + expect(rows.map((row) => row.key)).toEqual(["claude-sonnet-5", "gpt-5.5", ""]) + expect(rows[1]).toMatchObject({ + key: "gpt-5.5", + current: { sessions: 3, cost: 0.9 }, + previous: { sessions: 2, cost: 0.5 }, + }) + // A key the previous window never saw reads as zeros, not as a missing + // row — the client shows it as new rather than as a -100%. + expect(rows[0]).toMatchObject({ previous: { sessions: 0, cost: 0 } }) + } finally { + await harness.dispose() + } + }) + + it("reads a non-model dimension over the spans that can carry its key", async () => { + const { harness, readSql } = breakdownHarness([ + overviewRow({ period: "current", key: "run_tests", keyCount: 0, sessions: "4" }), + overviewRow({ period: "keys", key: "", keyCount: 2, sessions: "0" }), + ]) + + try { + const response = await harness.post("/internal/ai-sessions/overview/breakdown", { + ...WINDOW, + dimension: "tool", + }) + expect(response.status).toBe(200) + // The dimension picks the column AND the population: a tool key can + // only come from a tool call, as a model key can only come from a + // model call. + expect(readSql()).toContain("toString(ai_trace_index.ToolName) AS key") + expect(readSql()).toContain("AND ai_trace_index.IsToolCall = 1") + expect(readSql()).not.toContain("AND ai_trace_index.IsLlmCall = 1") + expect(response.body).toMatchObject({ dimension: "tool", totalKeys: 2 }) + } finally { + await harness.dispose() + } + }) + + it("refuses a limit past the table's cap with a 400 rather than a 500", async () => { + const harness = makeHarness({ compiledQuery: () => Effect.die("the read must never run") }) + + try { + const tooMany = await harness.post("/internal/ai-sessions/overview/breakdown", { + ...BREAKDOWN_BODY, + limit: AI_OVERVIEW_BREAKDOWN_MAX + 1, + }) + expect(tooMany.status).toBe(400) + // And a dimension that is not a column of the index at all. + const unknown = await harness.post("/internal/ai-sessions/overview/breakdown", { + ...WINDOW, + dimension: "customer", + }) + expect(unknown.status).toBe(400) + } finally { + await harness.dispose() + } + }) +}) + +describe("POST /internal/ai-sessions/overview/model-mix", () => { + const MODEL_MIX_BODY = { ...WINDOW, bucketSeconds: 300 } + + /** `count()` arrives quoted from a BYO-ClickHouse cluster and as a number + * from managed Tinybird; the row schema has to take both. */ + const ROWS = [ + { bucket: "2026-08-19T09:00:00.000Z", model: "gpt-5.5", llmCallSpans: "5" }, + { bucket: "2026-08-19T09:00:00.000Z", model: "claude-sonnet-5", llmCallSpans: 2 }, + { bucket: "2026-08-19T10:00:00.000Z", model: "gpt-5.5", llmCallSpans: "3" }, + ] + + const modelMixHarness = () => { + const contexts: Array = [] + let sql: string | undefined + const harness = makeHarness({ + compiledQuery: (_tenant, compiled, options) => { + contexts.push(options?.context) + sql = compiledQueryOf(compiled).sql + return compiledQueryOf(compiled).decodeRows(ROWS).pipe(Effect.orDie) + }, + }) + return { harness, contexts, readSql: () => sql ?? "" } + } + + it("counts the model-call spans of the window, bucket by bucket", async () => { + const { harness, contexts, readSql } = modelMixHarness() + + try { + const response = await harness.post("/internal/ai-sessions/overview/model-mix", MODEL_MIX_BODY) + expect(response.status).toBe(200) + expect(contexts).toEqual(["aiOverviewModelMix"]) + // One index read, cut at the width the caller asked for, over the + // model-call spans alone. + expect(readSql()).toContain("FROM ai_trace_index") + expect(readSql()).toContain("INTERVAL 300 SECOND") + expect(readSql()).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(readSql()).not.toContain("__PARAM_") + // The caller's window alone — no comparison band, so no second pair of + // bounds. + expect(readSql()).toContain(`Timestamp >= '${WINDOW.startTime}'`) + expect(readSql()).not.toContain("2026-08-19 07:00:00") + expect(response.body).toMatchObject({ + bucketSeconds: 300, + rows: [ + { bucket: "2026-08-19T09:00:00.000Z", model: "gpt-5.5", llmCallSpans: 5 }, + { bucket: "2026-08-19T09:00:00.000Z", model: "claude-sonnet-5", llmCallSpans: 2 }, + { bucket: "2026-08-19T10:00:00.000Z", model: "gpt-5.5", llmCallSpans: 3 }, + ], + }) + } finally { + await harness.dispose() + } + }) + + it("refuses a fractional bucket with a 400 rather than a 500", async () => { + const harness = makeHarness({ compiledQuery: () => Effect.die("the read must never run") }) + + try { + const response = await harness.post("/internal/ai-sessions/overview/model-mix", { + ...WINDOW, + bucketSeconds: 1.5, + }) + // `param.int` rejects a fraction inside the builder, which would be a + // 500 — the contract catches it at the boundary instead. + expect(response.status).toBe(400) + } finally { + await harness.dispose() + } + }) +}) diff --git a/apps/api/src/routes/internal/ai-sessions.http.ts b/apps/api/src/routes/internal/ai-sessions.http.ts index 3c5856675..9cec4024f 100644 --- a/apps/api/src/routes/internal/ai-sessions.http.ts +++ b/apps/api/src/routes/internal/ai-sessions.http.ts @@ -1,5 +1,8 @@ import { HttpApiBuilder } from "effect/unstable/httpapi" import { + AiOverviewBreakdownResponse, + AiOverviewModelMixResponse, + AiOverviewSummaryResponse, AiSessionTooLargeError, AI_SESSION_SPANS_MAX_SPANS, AI_SESSION_SUMMARY_MAX_TURNS, @@ -11,13 +14,15 @@ import { ListAiSessionsResponse, MapleInternalApi, MAX_AI_SESSION_SPANS_RESPONSE_BYTES, + type AiOverviewBreakdownRow, + type AiOverviewMeasures, type AiSessionTokenReporting, type AiSessionTokenTotals, type AiSessionTurnSummary, } from "@maple/domain/http" import { traceSessionTraceId } from "@maple/domain/gen-ai" import { Effect } from "effect" -import { CH } from "@maple/query-engine" +import { CH, formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" import * as Integrations from "@maple/query-engine-integrations" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" @@ -400,9 +405,242 @@ export const HttpAiSessionsInternalLive = HttpApiBuilder.group( return summary }), ) + .handle("overviewSummary", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.ai.overview.bucket_seconds": payload.bucketSeconds, + }) + // The comparison window is the caller's, shifted back by its own + // length: `previous` ends where `current` begins, so the two + // never overlap and the delta is over equal spans. + const params = { + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + ...previousWindow(payload.startTime, payload.endTime), + } + const selection = overviewSelection(payload) + // Two reads and not one: the tiles' percentiles cannot be folded + // from the chart's, so the window has to be grouped twice — and + // side by side that costs one read's latency rather than two. + const [totals, series] = yield* Effect.all( + [ + warehouse.compiledQuery( + tenant, + CH.compileUnion(Integrations.aiOverviewTotalsQuery(selection), params), + { context: "aiOverviewTotals" }, + ), + warehouse.compiledQuery( + tenant, + CH.compileUnion(Integrations.aiOverviewSeriesQuery(selection), { + ...params, + bucketSeconds: payload.bucketSeconds, + }), + { context: "aiOverviewSeries" }, + ), + ], + { concurrency: 2 }, + ) + const points = (period: Integrations.AiOverviewPeriod) => + series + .filter((row) => row.period === period) + .map((row) => ({ bucket: row.bucket, ...overviewMeasures(row) })) + yield* Effect.annotateCurrentSpan({ "maple.ai.overview.rows": series.length }) + return new AiOverviewSummaryResponse({ + bucketSeconds: payload.bucketSeconds, + // No row for a period means nothing ran in it: the branch + // grouped the window and found no sessions to group. The zeros + // stand in for it, which is what a client renders as "no + // comparison" rather than as a -100%. + current: overviewMeasures(totals.find((row) => row.period === "current")), + previous: overviewMeasures(totals.find((row) => row.period === "previous")), + series: points("current"), + previousSeries: points("previous"), + }) + }), + ) + .handle("overviewBreakdown", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.ai.overview.dimension": payload.dimension, + }) + const rows = yield* warehouse.compiledQuery( + tenant, + CH.compileUnion( + Integrations.aiOverviewBreakdownQuery({ + ...overviewSelection(payload), + dimension: payload.dimension, + limit: payload.limit, + }), + { + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + ...previousWindow(payload.startTime, payload.endTime), + }, + ), + { context: "aiOverviewBreakdown" }, + ) + const previous = new Map( + rows.filter((row) => row.period === "previous").map((row) => [row.key, row]), + ) + // The busiest first, and the ranking the query made is by sessions + // alone — cost orders the keys it tied. + const ranked = rows + .filter((row) => row.period === "current") + .sort((a, b) => b.sessions - a.sessions || b.cost - a.cost || a.key.localeCompare(b.key)) + const breakdown: ReadonlyArray = ranked.map((row) => ({ + key: row.key, + current: overviewMeasures(row), + // Zeros for a key that did not appear before, which reads as + // "new" rather than as a missing row. + previous: overviewMeasures(previous.get(row.key)), + })) + const totalKeys = rows.find((row) => row.period === "keys")?.keyCount ?? 0 + yield* Effect.annotateCurrentSpan({ + "maple.ai.overview.rows": breakdown.length, + "maple.ai.overview.total_keys": totalKeys, + }) + return new AiOverviewBreakdownResponse({ + dimension: payload.dimension, + rows: breakdown, + totalKeys, + }) + }), + ) + .handle("overviewModelMix", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + yield* Effect.annotateCurrentSpan({ + orgId: tenant.orgId, + "maple.ai.overview.bucket_seconds": payload.bucketSeconds, + }) + // No comparison window: the chart plots the selected window's + // bands and nothing behind them. + const rows = yield* warehouse.compiledQuery( + tenant, + CH.compile(Integrations.aiOverviewModelMixQuery(overviewSelection(payload)), { + orgId: tenant.orgId, + startTime: payload.startTime, + endTime: payload.endTime, + bucketSeconds: payload.bucketSeconds, + }), + { context: "aiOverviewModelMix" }, + ) + // The read folds its own tail, so the row cap is a guard the page + // cannot reach; a hit means the fold stopped bounding the response + // and the newest buckets are the ones missing. + yield* Effect.annotateCurrentSpan({ + "maple.ai.overview.rows": rows.length, + "maple.ai.overview.model_mix_capped": + rows.length >= Integrations.AI_OVERVIEW_MODEL_MIX_MAX_ROWS, + }) + return new AiOverviewModelMixResponse({ + bucketSeconds: payload.bucketSeconds, + rows: rows.map((row) => ({ + bucket: row.bucket, + model: row.model, + llmCallSpans: row.llmCallSpans, + })), + }) + }), + ) }), ) +/** + * The overview's selection, as both of its reads take it — the sessions list's + * counted filters, so the two pages measure the same sessions. + */ +const overviewSelection = (payload: { + readonly vendorIds?: ReadonlyArray + readonly serviceNames?: ReadonlyArray + readonly deploymentEnvs?: ReadonlyArray + readonly models?: ReadonlyArray + readonly agentNames?: ReadonlyArray + readonly toolNames?: ReadonlyArray + readonly hasErrors?: boolean +}) => ({ + vendorIds: payload.vendorIds, + serviceNames: payload.serviceNames, + deploymentEnvs: payload.deploymentEnvs, + models: payload.models, + agentNames: payload.agentNames, + toolNames: payload.toolNames, + hasErrors: payload.hasErrors, +}) + +/** + * The window of equal length ending where the caller's begins — the tiles' + * comparison. Computed here rather than asked for, so the delta cannot be + * quietly taken against a window of a different size. + * + * `prevEndTime` IS the caller's `startTime`: the read bounds the previous + * branch half-open (`[prevStartTime, prevEndTime)`), so the boundary second + * belongs to the current window alone and no session is measured in both. + * + * Both bounds are datetimes the request contract has already checked parse, so + * the arithmetic here cannot produce the `Invalid Date` a formatter throws on. + */ +const previousWindow = (startTime: string, endTime: string) => { + const start = parseWarehouseDateTime(startTime) + const span = parseWarehouseDateTime(endTime) - start + return { + prevStartTime: formatWarehouseDateTime(start - span), + prevEndTime: formatWarehouseDateTime(start), + } +} + +const NO_OVERVIEW_MEASURES: AiOverviewMeasures = { + sessions: 0, + erroredSessions: 0, + llmCalls: 0, + llmCallSpans: 0, + erroredLlmCalls: 0, + toolCalls: 0, + erroredToolCalls: 0, + cost: 0, + pricedLlmCalls: 0, + tokens: 0, + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + sessionDurationP50Ns: 0, + sessionDurationP95Ns: 0, +} + +/** One row's measures, or zeros for a period or a key that has no row. */ +const overviewMeasures = ( + row: Integrations.AiOverviewMeasuresOutput | undefined, +): AiOverviewMeasures => { + if (row === undefined) return NO_OVERVIEW_MEASURES + return { + sessions: row.sessions, + erroredSessions: row.erroredSessions, + llmCalls: row.llmCalls, + llmCallSpans: row.llmCallSpans, + erroredLlmCalls: row.erroredLlmCalls, + toolCalls: row.toolCalls, + erroredToolCalls: row.erroredToolCalls, + cost: row.cost, + pricedLlmCalls: row.pricedLlmCalls, + tokens: row.tokens, + inputTokens: row.inputTokens, + cacheReadTokens: row.cacheReadTokens, + cacheWriteTokens: row.cacheWriteTokens, + outputTokens: row.outputTokens, + reasoningTokens: row.reasoningTokens, + sessionDurationP50Ns: row.sessionDurationP50Ns, + sessionDurationP95Ns: row.sessionDurationP95Ns, + } +} + const NO_TOKENS: AiSessionTokenTotals = { input: 0, output: 0, cacheRead: 0 } const emptySummary = () => diff --git a/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts new file mode 100644 index 000000000..d6d2674da --- /dev/null +++ b/apps/api/src/services/warehouse/ai-overview.clickhouse.e2e.test.ts @@ -0,0 +1,687 @@ +// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted. +// Agent Sessions › Overview, against real rows. +// +// The overview's whole contract is that its numbers reconcile with the +// sessions LIST over the same window, and nothing about that can be proved +// from SQL text: +// +// - the SESSION a row belongs to is `max(SessionId)` per TRACE. Keyed per +// row instead, every span that carries no session id becomes its own +// session and every count is wrong by an order of magnitude behind a +// healthy 200. +// - USAGE is netted. A wrapper that rolls up its children's tokens, a +// gateway that files a second trace of the same call under the same +// session, and a provider retry beneath the call each have to count once, +// and each of those three is an array pass over real rows. +// - the buckets have to SUM to the totals, which is a statement about where +// a session that ran across a bucket boundary lands. +// +// So this suite seeds spans into `traces`, lets the real migration chain's +// `ai_trace_index_mv` materialize them, and runs the real compiled builders +// over the result — beside `aiSessionPageQuery` over the same window, which is +// what the numbers are checked against. It reads with +// `use_variant_as_common_type = 0`, the setting managed Tinybird runs, so a +// `UNION ALL` branch whose types only agree on a modern analyzer fails here. + +import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-clickhouse" +import { MAPLE_AI_SESSION_ID_ATTR, MAPLE_AI_VENDOR_ID_ATTR } from "@maple/domain/gen-ai" +import * as Integrations from "@maple/query-engine-integrations" +import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" +import { + ANALYZER_STRICTNESS, + applyRealMigrations, + clickhouseE2eEnabled, + clickhouseExec, + uniqueDatabase, +} from "./clickhouse-e2e-support" + +const database = uniqueDatabase("maple_ai_overview_e2e") +const ORG_ID = "org_ai_overview_e2e" +const FOREIGN_ORG_ID = "org_ai_overview_e2e_other" + +// Anchored to now: `traces` and `ai_trace_index` both enforce a 30-day TTL at +// insert, so a hardcoded date would one day drop every seed and leave the suite +// comparing nothing to nothing. +const HOUR_MS = 3_600_000 +const BASE_MS = Math.floor((Date.now() - 2 * HOUR_MS) / 1000) * 1000 + +/** Half an hour on — far enough to land in its own five-minute bucket. */ +const LATER_MS = BASE_MS + 1_800_000 + +/** Inside the comparison window, which ends where the caller's begins. */ +const EARLIER_MS = BASE_MS - 2 * HOUR_MS + +const SESSION_ID = `${ORG_ID}:overview-1` +/** The ordinary shape: a turn span that rolls up its two model calls, and a + * tool call that failed. Its GPT call failed too. */ +const TRACE_TURN = "aioverve2e00000000000000000000001" +/** The gateway's own trace of the first model call — same session, same + * response id, a price the app's SDK did not have, and the SAME failure: one + * call, netted, but two failed model-call spans. */ +const TRACE_MIRROR = "aioverve2e00000000000000000000002" +/** No session id anywhere, so the trace IS the session. Its model call failed. */ +const TRACE_SESSIONLESS = "aioverve2e00000000000000000000003" +/** A row written before the token buckets existed — inserted into the index + * directly, because the materialized view derives the buckets from the same + * attributes as the total and cannot produce one. */ +const TRACE_PRE_BUCKETS = "aioverve2e00000000000000000000004" +/** The comparison window's only session. */ +const TRACE_EARLIER = "aioverve2e00000000000000000000005" +const TRACE_FOREIGN = "aioverve2e00000000000000000000006" +/** More models than the mix plots bands for, half an hour PAST the window + * every other read here takes — so the fold has a population to fold and no + * other assertion has to account for it. */ +const TRACE_MODEL_TAIL = "aioverve2e00000000000000000000007" + +const GPT = "gpt-5" +const CLAUDE = "claude-sonnet-5" +/** One span each, so the ranking falls to the tie-break and the bands are + * `tail-model-1` … `tail-model-5` with the last two under `other`. */ +const TAIL_MODELS = [1, 2, 3, 4, 5, 6, 7].map((n) => `tail-model-${n}`) +/** Half an hour past the window's end. */ +const TAIL_MS = BASE_MS + HOUR_MS + 1_800_000 +/** The response id the app's SDK and the gateway both report for one call. */ +const SHARED_RESPONSE_ID = "resp-shared-1" + +interface SeedSpan { + readonly traceId: string + readonly spanId: string + readonly parentSpanId?: string + readonly name: string + readonly ms: number + readonly durationNs: number + readonly status: string + readonly attrs: Readonly> +} + +const agentSpan = (attrs: Readonly>) => ({ + [MAPLE_AI_VENDOR_ID_ATTR]: "eve", + ...attrs, +}) + +/** Tokens and a price, under the canonical semconv keys. */ +const usage = (input: number, output: number, cost: number, responseId?: string) => { + const base = { + "gen_ai.usage.input_tokens": String(input), + "gen_ai.usage.output_tokens": String(output), + "gen_ai.usage.cost": String(cost), + } + return responseId === undefined ? base : { ...base, "gen_ai.response.id": responseId } +} + +const SEED_SPANS: ReadonlyArray = [ + // The turn span carries the session id AND its children's usage summed onto + // it — the roll-up the netting has to cancel. + { + traceId: TRACE_TURN, + spanId: "overview-turn-1", + name: "invoke_agent slack-agent", + ms: BASE_MS, + durationNs: 10_000_000, + status: "Ok", + attrs: agentSpan({ + [MAPLE_AI_SESSION_ID_ATTR]: SESSION_ID, + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "slack-agent", + ...usage(120, 60, 0.03), + }), + }, + // The call the gateway mirrors below, and it FAILED — so the same failure is + // on the wire twice while the netting collapses the two spans into one call. + { + traceId: TRACE_TURN, + spanId: "overview-chat-gpt", + parentSpanId: "overview-turn-1", + name: "chat gpt-5", + ms: BASE_MS + 1, + durationNs: 4_000_000, + status: "Error", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(100, 50, 0.02, SHARED_RESPONSE_ID), + }), + }, + // A second model, in the same session — what makes the breakdown's rows + // overlap. + { + traceId: TRACE_TURN, + spanId: "overview-chat-claude", + parentSpanId: "overview-turn-1", + name: "chat claude-sonnet-5", + ms: BASE_MS + 4, + durationNs: 3_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": CLAUDE, + ...usage(20, 10, 0.01, "resp-claude-1"), + }), + }, + { + traceId: TRACE_TURN, + spanId: "overview-tool-1", + parentSpanId: "overview-chat-gpt", + name: "execute_tool search_traces", + ms: BASE_MS + 2, + durationNs: 1_000_000, + status: "Error", + attrs: agentSpan({ + "gen_ai.operation.name": "execute_tool", + "gen_ai.tool.name": "search_traces", + }), + }, + // The gateway's mirror: its own trace of the GPT call, under the same + // session id and the same response id, priced higher — and carrying the + // call's failure a second time. + { + traceId: TRACE_MIRROR, + spanId: "overview-chat-mirror", + name: "chat gpt-5", + ms: BASE_MS + 3, + durationNs: 5_000_000, + status: "Error", + attrs: agentSpan({ + [MAPLE_AI_SESSION_ID_ATTR]: SESSION_ID, + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(100, 50, 0.05, SHARED_RESPONSE_ID), + }), + }, + // A sessionless trace, half an hour later, whose model call failed. + { + traceId: TRACE_SESSIONLESS, + spanId: "overview-chat-late", + name: "chat claude-sonnet-5", + ms: LATER_MS, + durationNs: 6_000_000, + status: "Error", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": CLAUDE, + ...usage(200, 100, 0.1, "resp-late-1"), + }), + }, + // The comparison window's only session. + { + traceId: TRACE_EARLIER, + spanId: "overview-chat-earlier", + name: "chat gpt-5", + ms: EARLIER_MS, + durationNs: 2_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(10, 5, 0.01, "resp-earlier-1"), + }), + }, + // Seven models in one bucket, outside every other read's window: the mix + // plots five bands and counts the rest under `other`. + ...TAIL_MODELS.map((model, index) => ({ + traceId: TRACE_MODEL_TAIL, + spanId: `overview-chat-tail-${index}`, + name: `chat ${model}`, + ms: TAIL_MS + index, + durationNs: 1_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": model, + ...usage(1, 1, 0.001, `resp-tail-${index}`), + }), + })), +] + +/** Another org's session, in the same window — the reads must never see it. */ +const FOREIGN_SPAN: SeedSpan = { + traceId: TRACE_FOREIGN, + spanId: "overview-chat-foreign", + name: "chat gpt-5", + ms: BASE_MS + 5, + durationNs: 9_000_000, + status: "Ok", + attrs: agentSpan({ + "gen_ai.operation.name": "chat", + "gen_ai.response.model": GPT, + ...usage(999, 999, 9.99, "resp-foreign-1"), + }), +} + +const quote = (value: string): string => `'${value.replaceAll("'", "\\'")}'` +const chDateTime = (epochMs: number): string => new Date(epochMs).toISOString().replace("T", " ").slice(0, 23) +const chMap = (attrs: Readonly>): string => + `map(${Object.entries(attrs) + .flatMap(([key, value]) => [quote(key), quote(value)]) + .join(", ")})` + +const seed = async (): Promise => { + const rows = [ + ...SEED_SPANS.map((span) => [ORG_ID, span] as const), + [FOREIGN_ORG_ID, FOREIGN_SPAN] as const, + ] + .map( + ([orgId, span]) => + `(${quote(orgId)}, ${quote(chDateTime(span.ms))}, ${quote(span.traceId)}, ${quote(span.spanId)}, ${quote(span.parentSpanId ?? "")}, ${quote(span.name)}, 'Internal', 'agent-service', ${span.durationNs}, ${quote(span.status)}, 1, ${chMap(span.attrs)}, ${chMap({ "deployment.environment.name": "production" })})`, + ) + .join("\n,") + + await clickhouseExec( + `INSERT INTO traces + (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, SampleRate, SpanAttributes, ResourceAttributes) + VALUES\n${rows}`, + database, + ) + + // The pre-0031 row, written straight into the index: a total with five + // empty buckets, which is what every row materialized before the bucket + // columns existed still looks like. The overview must report the total and + // leave the buckets summing to nothing rather than inventing a split. + await clickhouseExec( + `INSERT INTO ai_trace_index + (OrgId, Timestamp, TraceId, SessionId, VendorId, ServiceName, DeploymentEnv, Model, AgentName, ToolName, + SpanId, ParentSpanId, Duration, IsError, IsLlmCall, IsToolCall, Tokens, Cost, ResponseId, VendorVersion, + InputTokens, CacheReadTokens, CacheWriteTokens, OutputTokens, ReasoningTokens) + VALUES (${quote(ORG_ID)}, ${quote(chDateTime(BASE_MS + 6))}, ${quote(TRACE_PRE_BUCKETS)}, '', 'eve', + 'agent-service', 'production', '', '', '', 'overview-chat-legacy', '', 2000000, 0, 1, 0, 500, 0, '', '', + 0, 0, 0, 0, 0)`, + database, + ) +} + +const runJson = async (sql: string): Promise>> => { + const body = await clickhouseExec(normalizeSqlForClickHouseClient(sql), database, { + default_format: "JSON", + output_format_json_quote_64bit_integers: "0", + ...ANALYZER_STRICTNESS, + }) + const parsed = JSON.parse(body) as { readonly data?: ReadonlyArray> } + return parsed.data ?? [] +} + +const window = { + orgId: ORG_ID, + startTime: chDateTime(BASE_MS - HOUR_MS), + endTime: chDateTime(BASE_MS + HOUR_MS), +} + +/** The comparison window the route computes: equal length, ending where the + * caller's begins. `TRACE_EARLIER` is the only session inside it. */ +const compareWindow = { + ...window, + prevStartTime: chDateTime(BASE_MS - 3 * HOUR_MS), + prevEndTime: chDateTime(BASE_MS - HOUR_MS), +} + +const totals = async (opts: Integrations.AiOverviewFilterOpts = {}) => { + const compiled = compileUnionUnsafe(Integrations.aiOverviewTotalsQuery(opts), compareWindow) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + return { + current: rows.find((row) => row.period === "current"), + previous: rows.find((row) => row.period === "previous"), + } +} + +/** The sessions list over the same window — what every number here is checked + * against. */ +const listRows = async (opts: Integrations.AiSessionPageOpts = {}) => { + const compiled = compileUnsafe(Integrations.aiSessionPageQuery(opts), window) + return Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) +} + +const sumOf = (rows: ReadonlyArray, read: (row: T) => number) => + rows.reduce((total, row) => total + read(row), 0) + +describe.skipIf(!clickhouseE2eEnabled)("agent overview reads", () => { + beforeAll(async () => { + await clickhouseExec(`CREATE DATABASE ${database}`) + await applyRealMigrations(database) + await seed() + }, 180_000) + + afterAll(async () => { + await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`) + }, 30_000) + + it("counts the sessions the list would have listed, and nobody else's", async () => { + const { current } = await totals() + const list = await listRows() + + // Three sessions: the vendor's own (two traces, the turn's and the + // gateway's mirror of it), the sessionless trace, and the pre-0031 row. + // The foreign org's session is in neither. + assert.strictEqual(current?.sessions, list.length) + assert.strictEqual(current?.sessions, 3) + assert.deepStrictEqual( + [...list].map((row) => row.sessionId).sort(), + [SESSION_ID, `trace:${TRACE_PRE_BUCKETS}`, `trace:${TRACE_SESSIONLESS}`].sort(), + ) + }) + + it("nets usage exactly as the list nets it — the roll-up, the mirror and the retry", async () => { + const { current } = await totals() + const list = await listRows() + + // The turn span reports its children's tokens as its own and the gateway + // reports the GPT call a second time: 180 raw tokens on the turn, 180 on + // its two model calls and 150 on the mirror, which is 510 summed and 180 + // netted — 150 for the GPT call (once, priced at the higher of the two + // claims) and 30 for the Claude call. + assert.strictEqual(current?.tokens, sumOf(list, (row) => row.totalTokens)) + assert.strictEqual(current?.tokens, 180 + 300 + 500) + assert.closeTo( + current!.cost, + sumOf(list, (row) => row.cost), + 1e-9, + ) + // 0.05 for the mirrored call (the gateway's price, not the SDK's), 0.01 + // for the Claude call, 0.10 for the late one; the roll-up adds nothing. + assert.closeTo(current!.cost, 0.16, 1e-9) + assert.strictEqual(current?.llmCalls, sumOf(list, (row) => row.llmCalls)) + assert.strictEqual(current?.llmCalls, 4) + // Coverage, not a price: every call but the pre-0031 one carried one. + assert.strictEqual(current?.pricedLlmCalls, 3) + assert.strictEqual(current?.toolCalls, sumOf(list, (row) => row.toolCalls)) + assert.strictEqual(current?.toolCalls, 1) + }) + + it("leaves a pre-bucket row's total whole and its buckets empty", async () => { + const { current } = await totals() + + // The five buckets are the disjoint split of the total for every row + // materialized since migration 0031, and zeros for the rows before it. + // Summing them to the total in SQL would invent a split the row never + // reported; the client falls back to the total instead. + assert.strictEqual(current?.inputTokens, 120 + 200) + assert.strictEqual(current?.outputTokens, 60 + 100) + assert.strictEqual(current?.cacheReadTokens, 0) + assert.strictEqual(current?.cacheWriteTokens, 0) + assert.strictEqual(current?.reasoningTokens, 0) + const buckets = + current!.inputTokens + + current!.cacheReadTokens + + current!.cacheWriteTokens + + current!.outputTokens + + current!.reasoningTokens + assert.strictEqual(buckets, 480) + assert.isBelow(buckets, current!.tokens) + }) + + it("counts failures against the population each belongs to", async () => { + const { current } = await totals() + const list = await listRows() + + // Two of the three sessions carry a failed agent span — the same two the + // list's own `hasErrors` matches. + assert.strictEqual(current?.erroredSessions, list.filter((row) => row.errorAgentSpans > 0).length) + assert.strictEqual(current?.erroredSessions, 2) + // One failed tool call, out of one. The list counts the DEEPEST failure + // (a failed tool whose child also failed is the child's echo) while this + // counts the failed tool spans; the two agree here because no failed span + // sits under the failed tool. + assert.strictEqual(current?.erroredToolCalls, 1) + assert.strictEqual(current?.erroredToolCalls, sumOf(list, (row) => row.toolErrors)) + + // The GPT call failed and the gateway mirrored that failure into its own + // trace, so three of the five model-call SPANS failed — while those five + // spans net to four calls. The rate is the failures over the population + // they were counted in, which cannot pass 100%; over `llmCalls` the + // mirrored call would be counted twice against itself. + assert.strictEqual(current?.erroredLlmCalls, 3) + assert.strictEqual(current?.llmCallSpans, 5) + assert.strictEqual(current?.llmCalls, 4) + assert.closeTo(current!.erroredLlmCalls / current!.llmCallSpans, 3 / 5, 1e-9) + assert.isAtMost(current!.erroredLlmCalls / current!.llmCallSpans, 1) + + // And the filter selects exactly those sessions. + const failing = await totals({ hasErrors: true }) + assert.strictEqual(failing.current?.sessions, 2) + assert.strictEqual(failing.current?.erroredSessions, 2) + }) + + it("measures the session's extent, first agent span to last", async () => { + const { current } = await totals() + const list = await listRows() + + // The session's extent is its first span to its last span's END: 10ms for + // the turn's session (its own span outlives every call beneath it), 6ms + // and 2ms for the other two — the same three the list reports. + const extents = [...list].map((row) => row.agentDurationMs).sort((a, b) => a - b) + assert.deepStrictEqual(extents, [2, 6, 10]) + assert.strictEqual(current?.sessionDurationP50Ns, extents[1]! * 1_000_000) + assert.isAbove(current!.sessionDurationP95Ns, extents[1]! * 1_000_000) + assert.isAtMost(current!.sessionDurationP95Ns, extents[2]! * 1_000_000) + }) + + it("measures the window before the caller's in the same read", async () => { + const { previous } = await totals() + + // One session, an hour before the window opens. Nothing of the current + // window leaks into it. + assert.strictEqual(previous?.sessions, 1) + assert.strictEqual(previous?.tokens, 15) + assert.closeTo(previous!.cost, 0.01, 1e-9) + assert.strictEqual(previous?.toolCalls, 0) + }) + + it("files a session under the bucket it started in, so the buckets sum to the totals", async () => { + const compiled = compileUnionUnsafe(Integrations.aiOverviewSeriesQuery(), { + ...compareWindow, + bucketSeconds: 300, + }) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + const series = rows.filter((row) => row.period === "current") + const { current } = await totals() + + // Two buckets: the turn's session and the pre-0031 row in the first, the + // sessionless trace half an hour later in its own. + assert.strictEqual(series.length, 2) + assert.isTrue(series[0]!.bucket < series[1]!.bucket, `${series[0]!.bucket} < ${series[1]!.bucket}`) + assert.strictEqual( + sumOf(series, (row) => row.sessions), + current?.sessions, + ) + assert.strictEqual( + sumOf(series, (row) => row.tokens), + current?.tokens, + ) + assert.closeTo( + sumOf(series, (row) => row.cost), + current!.cost, + 1e-9, + ) + assert.strictEqual( + sumOf(series, (row) => row.llmCalls), + current?.llmCalls, + ) + // The session the gateway mirrored spans both its traces and still lands + // in one bucket — the one its first span started in. + assert.strictEqual(series[0]?.sessions, 2) + assert.strictEqual(rows.filter((row) => row.period === "previous").length, 1) + }) + + it("files a session under every model it used, and its tokens under the call that reported them", async () => { + const compiled = compileUnionUnsafe( + Integrations.aiOverviewBreakdownQuery({ dimension: "model" }), + compareWindow, + ) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + const byKey = new Map(rows.filter((row) => row.period === "current").map((row) => [row.key, row])) + const { current } = await totals() + + // The turn's session used two models, so it is a session under each — + // rows overlap and do not sum to the totals. + assert.deepStrictEqual([...byKey.keys()].sort(), ["", CLAUDE, GPT]) + assert.strictEqual(byKey.get(GPT)?.sessions, 1) + assert.strictEqual(byKey.get(CLAUDE)?.sessions, 2) + assert.isAbove( + sumOf([...byKey.values()], (row) => row.sessions), + current!.sessions, + ) + + // The usage does NOT overlap: a call's tokens are charged to the model + // that reported them, with the mirror still collapsed onto one claim and + // the turn span's roll-up — which names no model — never counted. + assert.strictEqual(byKey.get(GPT)?.tokens, 150) + assert.closeTo(byKey.get(GPT)!.cost, 0.05, 1e-9) + assert.strictEqual(byKey.get(CLAUDE)?.tokens, 30 + 300) + assert.closeTo(byKey.get(CLAUDE)!.cost, 0.11, 1e-9) + // And the mirror is where the two model-call populations part: under GPT, + // two failed spans over two spans, which net to one call. A rate taken + // against the netted call would read 200%. + assert.strictEqual(byKey.get(GPT)?.erroredLlmCalls, 2) + assert.strictEqual(byKey.get(GPT)?.llmCallSpans, 2) + assert.strictEqual(byKey.get(GPT)?.llmCalls, 1) + + // The pre-0031 row names no model and is the unattributed key, not a gap. + assert.strictEqual(byKey.get("")?.tokens, 500) + assert.strictEqual( + sumOf([...byKey.values()], (row) => row.tokens), + current?.tokens, + ) + + // The previous window is measured over the same keys, and the third + // branch counts what the table is not showing. + const previous = rows.filter((row) => row.period === "previous") + assert.deepStrictEqual( + previous.map((row) => row.key), + [GPT], + ) + assert.strictEqual(previous[0]?.tokens, 15) + assert.strictEqual(rows.find((row) => row.period === "keys")?.keyCount, 3) + }) + + it("reads a tool breakdown over tool calls and an agent breakdown over every span", async () => { + const read = async (dimension: Integrations.AiOverviewBreakdownOpts["dimension"]) => { + const compiled = compileUnionUnsafe( + Integrations.aiOverviewBreakdownQuery({ dimension }), + compareWindow, + ) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + return rows.filter((row) => row.period === "current") + } + + // One tool, called once, and it failed. A tool span reports no usage, so + // its row costs nothing — which is the honest answer, not a missing one. + const tools = await read("tool") + assert.deepStrictEqual( + tools.map((row) => ({ key: row.key, toolCalls: row.toolCalls, errored: row.erroredToolCalls })), + [{ key: "search_traces", toolCalls: 1, errored: 1 }], + ) + assert.strictEqual(tools[0]?.cost, 0) + + // Agent names sit on the turn span alone, so the rest of the spans key + // under '' — the unattributed row the page shows beside the named one. + const agents = await read("agent") + assert.deepStrictEqual([...agents].map((row) => row.key).sort(), ["", "slack-agent"]) + + // Every agent span carries a service and a vendor, so those two never + // have an unattributed row. + const services = await read("service") + assert.deepStrictEqual( + services.map((row) => ({ key: row.key, sessions: row.sessions })), + [{ key: "agent-service", sessions: 3 }], + ) + }) + + it("splits the window's model-call spans by model, without netting a single one", async () => { + const modelMix = async (opts: Integrations.AiOverviewFilterOpts = {}) => { + const compiled = compileUnsafe(Integrations.aiOverviewModelMixQuery(opts), { + ...window, + bucketSeconds: 300, + }) + return Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + } + + const rows = await modelMix() + const { current } = await totals() + + // Two buckets, half an hour apart, and the busiest model of a bucket + // first. The gateway's mirror is a SPAN of its own here — the netting + // that makes it one call never runs — so GPT carries two. + assert.deepStrictEqual( + rows.map((row) => ({ model: row.model, spans: row.llmCallSpans })), + [ + { model: GPT, spans: 2 }, + { model: CLAUDE, spans: 1 }, + { model: CLAUDE, spans: 1 }, + ], + ) + assert.strictEqual(rows[0]!.bucket, rows[1]!.bucket) + assert.isTrue(rows[1]!.bucket < rows[2]!.bucket, `${rows[1]!.bucket} < ${rows[2]!.bucket}`) + + // The tool call is not a model call, and the pre-0031 row is a model call + // that named no model — so the mix is exactly the summary's SPAN + // population less that one row. GPT's two spans against the one call they + // net to is the whole difference between this read and the breakdown. + assert.strictEqual( + sumOf(rows, (row) => row.llmCallSpans), + current!.llmCallSpans - 1, + ) + assert.strictEqual( + sumOf(rows, (row) => row.llmCallSpans), + 4, + ) + assert.isFalse(rows.some((row) => row.model === "" || row.model === "search_traces")) + + // A model filter is the per-trace existence test every other overview + // read applies: it drops the sessionless trace, which never called GPT, + // and keeps every model span of the traces it selected — so Claude is + // still a band under a GPT filter. + const gptOnly = await modelMix({ models: [GPT] }) + assert.deepStrictEqual( + gptOnly.map((row) => ({ model: row.model, spans: row.llmCallSpans })), + [ + { model: GPT, spans: 2 }, + { model: CLAUDE, spans: 1 }, + ], + ) + assert.strictEqual(gptOnly[0]!.bucket, rows[0]!.bucket) + }) + + it("counts every model past the busiest five under one band", async () => { + // The tail trace's own window: seven models, one span each, in a single + // bucket. Folded client-side this is seven rows a bucket and the row cap + // would one day cut the newest bucket off the chart. + const compiled = compileUnsafe(Integrations.aiOverviewModelMixQuery(), { + orgId: ORG_ID, + startTime: chDateTime(TAIL_MS - 60_000), + endTime: chDateTime(TAIL_MS + 60_000), + bucketSeconds: 300, + }) + const rows = Effect.runSync(compiled.decodeRows(await runJson(compiled.sql))) + + // Six rows and not seven: the five bands the ranking kept — ties broken by + // name, so the bands are stable between loads — and `other` for the rest. + assert.strictEqual(rows.length, 6) + assert.deepStrictEqual( + [...rows].map((row) => row.model).sort(), + [...TAIL_MODELS.slice(0, 5), "other"].sort(), + ) + assert.strictEqual(rows.find((row) => row.model === "other")?.llmCallSpans, 2) + assert.strictEqual( + sumOf(rows, (row) => row.llmCallSpans), + TAIL_MODELS.length, + ) + }) + + it("selects sessions the way the list selects them, by any span of the trace", async () => { + // A model filter and a tool filter together: they are matched by + // DIFFERENT spans of the same trace, which a row predicate could never + // do — and the session the two select is the one the list selects. + const { current } = await totals({ models: [GPT], toolNames: ["search_traces"] }) + const list = await listRows({ models: [GPT], toolNames: ["search_traces"] }) + + assert.strictEqual(current?.sessions, list.length) + assert.strictEqual(current?.sessions, 1) + assert.strictEqual(current?.tokens, sumOf(list, (row) => row.totalTokens)) + + // A filter no span carries selects nothing, rather than everything. + const none = await totals({ vendorIds: ["vercel_ai_sdk"] }) + assert.strictEqual(none.current?.sessions, 0) + assert.strictEqual(none.current?.cost, 0) + assert.strictEqual(none.current?.sessionDurationP50Ns, 0) + }) +}) diff --git a/apps/web/src/api/warehouse/ai-agent-overview.test.ts b/apps/web/src/api/warehouse/ai-agent-overview.test.ts new file mode 100644 index 000000000..b4dbc253e --- /dev/null +++ b/apps/web/src/api/warehouse/ai-agent-overview.test.ts @@ -0,0 +1,162 @@ +import { Effect, Exit } from "effect" +import { describe, expect, it } from "vitest" + +import { AI_OVERVIEW_FILTER_VALUE_MAX_LENGTH, AiOverviewMeasures } from "@maple/domain/http" + +import { + AiOverviewBucketedInput, + mapOverviewBreakdown, + mapOverviewMeasures, + mapOverviewModelMix, + mapOverviewSeries, + selectionFields, +} from "./ai-agent-overview" +import { WarehouseDecodeError, decodeInput } from "./effect-utils" + +const wire = (overrides: Partial = {}): AiOverviewMeasures => ({ + sessions: 10, + erroredSessions: 1, + llmCalls: 80, + llmCallSpans: 94, + erroredLlmCalls: 3, + toolCalls: 63, + erroredToolCalls: 2, + cost: 2.64, + pricedLlmCalls: 88, + tokens: 674_000, + inputTokens: 152_000, + cacheReadTokens: 373_000, + cacheWriteTokens: 27_000, + outputTokens: 81_000, + reasoningTokens: 41_000, + sessionDurationP50Ns: 42_000_000_000, + sessionDurationP95Ns: 96_000_000_000, + ...overrides, +}) + +describe("mapOverviewMeasures", () => { + it("converts the session quantiles from nanoseconds to milliseconds", () => { + const row = mapOverviewMeasures(wire()) + expect(row.sessionDurationP50Ms).toBe(42_000) + expect(row.sessionDurationP95Ms).toBe(96_000) + }) + + it("carries the raw span population separately from the netted volume", () => { + const row = mapOverviewMeasures(wire()) + expect(row.llmCalls).toBe(80) + expect(row.llmCallSpans).toBe(94) + }) +}) + +describe("mapOverviewSeries", () => { + it("reads a bucket as UTC rather than as local time", () => { + const [point] = mapOverviewSeries([{ bucket: "2026-09-10T12:00:00.000Z", ...wire() }]) + expect(point.bucket).toBe(Date.UTC(2026, 8, 10, 12, 0, 0)) + }) +}) + +describe("mapOverviewBreakdown", () => { + it("keeps both windows per key, and `''` as a real key", () => { + const [row] = mapOverviewBreakdown([{ key: "", current: wire(), previous: wire({ sessions: 4 }) }]) + expect(row.key).toBe("") + expect(row.current.sessions).toBe(10) + expect(row.previous.sessions).toBe(4) + }) +}) + +describe("mapOverviewModelMix", () => { + it("reads the bucket as UTC and leaves the band alone", () => { + // `other` is a band the read itself folds, and reaches the chart as the + // key the client folds its own tail into. + const rows = mapOverviewModelMix([ + { bucket: "2026-09-10T18:00:00.000Z", model: "claude-opus-5", llmCallSpans: 42 }, + { bucket: "2026-09-10T18:00:00.000Z", model: "other", llmCallSpans: 7 }, + ]) + expect(rows[0].bucket).toBe(Date.UTC(2026, 8, 10, 18, 0, 0)) + expect(rows[0].model).toBe("claude-opus-5") + expect(rows[0].llmCallSpans).toBe(42) + expect(rows[1].model).toBe("other") + }) +}) + +/** + * The local input schemas exist to mirror the domain request's bounds: a + * violation has to land as a `WarehouseDecodeError` the page renders, because + * the alternative is `new AiOverview*Request` throwing inside the read and + * taking the page with it. A bound the mirror omits is exactly that defect. + */ +describe("the domain's bounds, mirrored", () => { + const WINDOW = { startTime: "2026-09-10 00:00:00", endTime: "2026-09-10 03:00:00" } + const bucketed = (overrides: Record = {}) => ({ + ...WINDOW, + bucketSeconds: 300, + ...overrides, + }) + + const decode = (data: unknown) => + Effect.runSyncExit(decodeInput(AiOverviewBucketedInput, data, "aiOverviewSummary")) + + const failure = (data: unknown) => + Effect.runSync(Effect.flip(decodeInput(AiOverviewBucketedInput, data, "aiOverviewSummary"))) + + it("takes the board's own selection", () => { + expect(Exit.isSuccess(decode(bucketed({ model: "claude-opus-5", hasErrors: true })))).toBe(true) + }) + + it("fails typed for a filter value past the contract's per-value cap", () => { + const error = failure(bucketed({ model: "m".repeat(AI_OVERVIEW_FILTER_VALUE_MAX_LENGTH + 1) })) + expect(error).toBeInstanceOf(WarehouseDecodeError) + expect(error.operation).toBe("aiOverviewSummary") + // One character under it is a value the contract accepts. + expect( + Exit.isSuccess(decode(bucketed({ model: "m".repeat(AI_OVERVIEW_FILTER_VALUE_MAX_LENGTH) }))), + ).toBe(true) + }) + + it("fails typed for a datetime the pattern admits and the calendar does not", () => { + expect(failure(bucketed({ startTime: "2026-13-45 99:99:99" }))).toBeInstanceOf(WarehouseDecodeError) + }) + + it("fails typed for an inverted window", () => { + expect( + failure({ startTime: WINDOW.endTime, endTime: WINDOW.startTime, bucketSeconds: 300 }), + ).toBeInstanceOf(WarehouseDecodeError) + }) +}) + +describe("selectionFields", () => { + const WINDOW = { startTime: "2026-09-10 00:00:00", endTime: "2026-09-10 03:00:00" } + + it("widens each single value into the array-valued key the contract takes", () => { + expect( + selectionFields({ + ...WINDOW, + framework: "eve", + service: "api", + environment: "prd", + model: "claude-opus-5", + agent: "captain", + tool: "run_tests", + hasErrors: true, + }), + ).toEqual({ + vendorIds: ["eve"], + serviceNames: ["api"], + deploymentEnvs: ["prd"], + models: ["claude-opus-5"], + agentNames: ["captain"], + toolNames: ["run_tests"], + hasErrors: true, + }) + }) + + it("omits the key a dimension has no value for, rather than sending an empty array", () => { + // An explicit `undefined` is not an absent key on the wire, and an empty + // `IN ()` list selects nothing rather than everything. + expect(selectionFields(WINDOW)).toEqual({}) + expect(selectionFields({ ...WINDOW, model: "claude-opus-5" })).toEqual({ + models: ["claude-opus-5"], + }) + expect(selectionFields({ ...WINDOW, hasErrors: false })).toEqual({ hasErrors: false }) + }) +}) diff --git a/apps/web/src/api/warehouse/ai-agent-overview.ts b/apps/web/src/api/warehouse/ai-agent-overview.ts new file mode 100644 index 000000000..9858b7999 --- /dev/null +++ b/apps/web/src/api/warehouse/ai-agent-overview.ts @@ -0,0 +1,265 @@ +// The three warehouse reads behind `/agent-sessions/overview`, and the only +// place in the web app that sees their wire shape. +// +// Two conversions happen here and nowhere else. Buckets arrive as ISO-8601 with +// a literal `Z`; `toEpochMs` reads them as UTC, where `new Date(value)` would +// read a bare warehouse datetime as local time. Durations arrive in +// nanoseconds and leave in milliseconds, because every formatter downstream +// takes milliseconds. +// +// The page's filters are single-valued — one model, one agent, one tool — and +// the contract takes arrays, so `selectionFields` widens them for all three of +// these reads. The Top sessions table reads the sessions list instead, and +// widens its own in `overviewSessionsInput`. + +import { Effect, Schema } from "effect" +import { + AI_OVERVIEW_BREAKDOWN_MAX, + AI_OVERVIEW_FILTER_VALUE_MAX_LENGTH, + AiOverviewBreakdownRequest, + AiOverviewDimension, + AiOverviewModelMixRequest, + AiOverviewSummaryRequest, + BucketSeconds, + isAiOverviewWindow, + type AiOverviewBreakdownRow, + type AiOverviewMeasures, + type AiOverviewModelMixPoint, + type AiOverviewSeriesPoint, +} from "@maple/domain/http" +import { toEpochMs } from "@maple/ui/lib/time-format" + +import type { + OverviewBreakdownEntry, + OverviewMeasurePoint, + OverviewMeasures, + OverviewModelMixRow, +} from "@/lib/agent-sessions/overview-analytics" +import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" + +import { WarehouseDateTimeString, decodeInput, runWarehouseQuery } from "./effect-utils" + +/** + * Every bound here MIRRORS the domain request's, and all three inputs carry + * every one of them: `decodeInput` turns a violation into a + * `WarehouseDecodeError` the page can render, where the domain constructor + * throws — a defect that would take the page down rather than fail one read. + * A bound that is mirrored loosely is the same defect with extra steps, which + * is why the value cap and the window rule are the domain's own. + */ +const overviewWindowValid = Schema.makeFilter( + (input: { readonly startTime: string; readonly endTime: string }) => + isAiOverviewWindow(input.startTime, input.endTime), + { identifier: "OverviewWindowValid" }, +) + +/** One value per dimension here; the request widens it into the array the + * contract takes, where it is capped at exactly this length. */ +const OverviewFilterValue = Schema.optional( + Schema.String.check(Schema.isMaxLength(AI_OVERVIEW_FILTER_VALUE_MAX_LENGTH)), +) + +/** + * The page's selection, as all three reads take it. + * + * Sent identically to every one of them so the numbers can never disagree about + * what they are counting: a summary narrower than the breakdown would make the + * tiles and the table tell different stories about one window. + */ +const aiOverviewSelectionFields = { + startTime: WarehouseDateTimeString, + endTime: WarehouseDateTimeString, + /** The SDK or gateway — `vendorIds` on the wire. */ + framework: OverviewFilterValue, + model: OverviewFilterValue, + agent: OverviewFilterValue, + service: OverviewFilterValue, + environment: OverviewFilterValue, + tool: OverviewFilterValue, + hasErrors: Schema.optional(Schema.Boolean), +} + +// The window rule is re-applied per input rather than inherited: spreading a +// struct's `fields` carries the fields and not its checks. +export const AiOverviewSelection = Schema.Struct(aiOverviewSelectionFields).check(overviewWindowValid) +export type AiOverviewSelection = Schema.Schema.Type + +export const AiOverviewBucketedInput = Schema.Struct({ + ...aiOverviewSelectionFields, + /** The domain's own bound: a fraction reaches `toStartOfInterval` as an + * `INTERVAL n SECOND` literal, which the builder refuses. */ + bucketSeconds: BucketSeconds, +}).check(overviewWindowValid) +export type AiOverviewBucketedInput = Schema.Schema.Type + +export const AiOverviewBreakdownInput = Schema.Struct({ + ...aiOverviewSelectionFields, + dimension: AiOverviewDimension, + limit: Schema.optional( + Schema.Number.check( + Schema.isInt(), + Schema.isBetween({ minimum: 1, maximum: AI_OVERVIEW_BREAKDOWN_MAX }), + ), + ), +}).check(overviewWindowValid) +export type AiOverviewBreakdownInput = Schema.Schema.Type + +/** The selection minus the window, widened into a request payload: the page + * filters by one value per dimension and the contract takes arrays. */ +export const selectionFields = (input: AiOverviewSelection) => ({ + ...(input.framework !== undefined && { vendorIds: [input.framework] }), + ...(input.service !== undefined && { serviceNames: [input.service] }), + ...(input.environment !== undefined && { deploymentEnvs: [input.environment] }), + ...(input.model !== undefined && { models: [input.model] }), + ...(input.agent !== undefined && { agentNames: [input.agent] }), + ...(input.tool !== undefined && { toolNames: [input.tool] }), + ...(input.hasErrors !== undefined && { hasErrors: input.hasErrors }), +}) + +/* ------------------------------------------------------------------------------------------------- + * Mappers + * -----------------------------------------------------------------------------------------------*/ + +const NS_PER_MS = 1_000_000 + +export function mapOverviewMeasures(row: AiOverviewMeasures): OverviewMeasures { + return { + sessions: row.sessions, + erroredSessions: row.erroredSessions, + llmCalls: row.llmCalls, + llmCallSpans: row.llmCallSpans, + erroredLlmCalls: row.erroredLlmCalls, + toolCalls: row.toolCalls, + erroredToolCalls: row.erroredToolCalls, + cost: row.cost, + pricedLlmCalls: row.pricedLlmCalls, + tokens: row.tokens, + inputTokens: row.inputTokens, + cacheReadTokens: row.cacheReadTokens, + cacheWriteTokens: row.cacheWriteTokens, + outputTokens: row.outputTokens, + reasoningTokens: row.reasoningTokens, + sessionDurationP50Ms: row.sessionDurationP50Ns / NS_PER_MS, + sessionDurationP95Ms: row.sessionDurationP95Ns / NS_PER_MS, + } +} + +export function mapOverviewSeries( + rows: ReadonlyArray, +): ReadonlyArray { + return rows.map((row) => ({ bucket: toEpochMs(row.bucket), ...mapOverviewMeasures(row) })) +} + +export function mapOverviewBreakdown( + rows: ReadonlyArray, +): ReadonlyArray { + return rows.map((row) => ({ + key: row.key, + current: mapOverviewMeasures(row.current), + previous: mapOverviewMeasures(row.previous), + })) +} + +export function mapOverviewModelMix( + rows: ReadonlyArray, +): ReadonlyArray { + return rows.map((row) => ({ + bucket: toEpochMs(row.bucket), + model: row.model, + llmCallSpans: row.llmCallSpans, + })) +} + +/* ------------------------------------------------------------------------------------------------- + * The reads + * -----------------------------------------------------------------------------------------------*/ + +/** The window and the one before it, whole and bucketed — the tiles and the grid. */ +export const getAiOverviewSummary = Effect.fn("AiSessions.aiOverviewSummary")(function* ({ + data, +}: { + data: AiOverviewBucketedInput +}) { + const input = yield* decodeInput(AiOverviewBucketedInput, data, "aiOverviewSummary") + yield* Effect.annotateCurrentSpan("maple.ai.overview.bucket_seconds", input.bucketSeconds) + const result = yield* runWarehouseQuery("aiOverviewSummary", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.aiSessionsInternal.overviewSummary({ + payload: new AiOverviewSummaryRequest({ + startTime: input.startTime, + endTime: input.endTime, + bucketSeconds: input.bucketSeconds, + ...selectionFields(input), + }), + }) + }), + ) + return { + bucketSeconds: result.bucketSeconds, + current: mapOverviewMeasures(result.current), + // Zeros where nothing ran then, which the tiles render as "no comparison" + // rather than as a -100%. + previous: mapOverviewMeasures(result.previous), + series: mapOverviewSeries(result.series), + previousSeries: mapOverviewSeries(result.previousSeries), + } +}) + +/** One dimension's busiest keys, each over both windows. */ +export const getAiOverviewBreakdown = Effect.fn("AiSessions.aiOverviewBreakdown")(function* ({ + data, +}: { + data: AiOverviewBreakdownInput +}) { + const input = yield* decodeInput(AiOverviewBreakdownInput, data, "aiOverviewBreakdown") + // Six of these run per board, one per dimension, so the span says which. + yield* Effect.annotateCurrentSpan("maple.ai.overview.dimension", input.dimension) + const result = yield* runWarehouseQuery("aiOverviewBreakdown", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.aiSessionsInternal.overviewBreakdown({ + payload: new AiOverviewBreakdownRequest({ + startTime: input.startTime, + endTime: input.endTime, + dimension: input.dimension, + ...(input.limit !== undefined && { limit: input.limit }), + ...selectionFields(input), + }), + }) + }), + ) + return { + dimension: result.dimension, + entries: mapOverviewBreakdown(result.rows), + totalKeys: result.totalKeys, + } +}) + +/** Model-call spans per bucket per model — the 100% stack. Current window only. */ +export const getAiOverviewModelMix = Effect.fn("AiSessions.aiOverviewModelMix")(function* ({ + data, +}: { + data: AiOverviewBucketedInput +}) { + const input = yield* decodeInput(AiOverviewBucketedInput, data, "aiOverviewModelMix") + yield* Effect.annotateCurrentSpan("maple.ai.overview.bucket_seconds", input.bucketSeconds) + const result = yield* runWarehouseQuery("aiOverviewModelMix", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.aiSessionsInternal.overviewModelMix({ + payload: new AiOverviewModelMixRequest({ + startTime: input.startTime, + endTime: input.endTime, + bucketSeconds: input.bucketSeconds, + ...selectionFields(input), + }), + }) + }), + ) + return { rows: mapOverviewModelMix(result.rows) } +}) + +export type AiOverviewSummaryData = Effect.Success> +export type AiOverviewBreakdownData = Effect.Success> +export type AiOverviewModelMixData = Effect.Success> diff --git a/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx b/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx new file mode 100644 index 000000000..0221cf9f4 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/agent-overview-view.test.tsx @@ -0,0 +1,344 @@ +// @vitest-environment jsdom +// TEST-SEAM: the router has no instance-level injection seam, so `Link` is +// replaced at the module boundary. What is under test is the page's own wiring +// — which control writes which search param, which row links where, and what +// the sections say about the data they are given. + +import { useState } from "react" +import { cleanup, fireEvent, render, screen, within } from "@testing-library/react" +import { afterEach, describe, expect, it, vi } from "vitest" + +import { WarehouseQueryError } from "@/api/warehouse/effect-utils" +import { buildOverviewFixture, type OverviewFixture } from "@/lab/agent-overview-fixture" +import { + buildAgentOverviewData, + type AgentOverviewData, +} from "@/lib/agent-sessions/overview-analytics" +import { + EMPTY_OVERVIEW_FACETS, + compareEnabled, + type AgentOverviewSearch, +} from "@/lib/agent-sessions/overview-search" +import type { OverviewTopSessionTab } from "@/lib/agent-sessions/use-agent-overview" + +import { AgentOverviewView, type AgentOverviewErrors } from "./agent-overview-view" + +// TEST-SEAM: the plots themselves are a canvas/ResizeObserver story jsdom cannot +// tell. What the cell puts around them — title, headline, unit, delta, legend — +// is real, and the legend is where the previous-period ghost shows up. +vi.mock("./overview-small-multiple", () => ({ + OVERVIEW_PLOT_HEIGHT: 104, + OverviewSmallMultiple: ({ chartId }: { chartId: string }) =>
, +})) + +vi.mock("@tanstack/react-router", () => ({ + Link: ({ children, to, params, search, ...props }: React.PropsWithChildren>) => ( + )} + > + {children} + + ), +})) + +const NOW = Date.UTC(2026, 8, 10, 12, 0, 0) + +/** The page's own shape: the Top sessions tab is the route's state, not the table's. */ +function Board({ + search, + onSearchChange, + data, + fixture, + windowLabel, + errors, + onRetry, +}: { + search: AgentOverviewSearch + onSearchChange: (patch: Partial) => void + data?: AgentOverviewData + fixture: OverviewFixture + windowLabel: string + errors?: AgentOverviewErrors + onRetry?: () => void +}) { + const [tab, setTab] = useState("cost") + return ( + + ) +} + +function renderView( + search: AgentOverviewSearch = {}, + scenario: "healthy7d" | "regression24h" = "regression24h", + errors?: AgentOverviewErrors, +) { + const fixture = buildOverviewFixture(scenario, NOW) + const data = buildAgentOverviewData({ ...fixture.input, compare: compareEnabled(search) }) + const onSearchChange = vi.fn() + render( + , + ) + return { onSearchChange, data, fixture } +} + +/** A read that failed, carrying the retryable body the panel reads its copy + * and its action from. */ +const readFailure = () => + new WarehouseQueryError({ operation: "aiOverviewSummary", message: "the warehouse said no" }) + +/** The section a heading owns — the page repeats labels across sections. */ +const sectionOf = (heading: string) => screen.getByRole("heading", { name: heading }).closest("section")! + +afterEach(cleanup) + +describe("AgentOverviewView", () => { + it("renders the seven KPI tiles and the nine small multiples", () => { + const { data } = renderView() + for (const tile of data.tiles) { + expect(screen.getAllByText(tile.label).length).toBeGreaterThan(0) + } + expect(document.querySelectorAll("[data-chart]")).toHaveLength(9) + expect(document.querySelectorAll("[data-plot]")).toHaveLength(9) + }) + + it("draws the previous-period ghost only while the comparison is on", () => { + // The fixture's window is a ragged number of buckets long, as the page's + // own default is, so this only passes while the ghost is shifted onto the + // bucket grid rather than by the window's raw length. + renderView({}) + expect(screen.getAllByText("prev").length).toBeGreaterThan(0) + cleanup() + renderView({ compare: false }) + expect(screen.queryByText("prev")).toBeNull() + }) + + it("ranks nothing on the rail while the comparison is off, and says why", () => { + const { data } = renderView({ compare: false }) + expect(data.movers).toEqual([]) + const rail = screen.getByRole("heading", { name: "What changed" }).closest("aside")! + expect(within(rail).getByText("Turn on compare to rank what changed.")).toBeTruthy() + // The coverage block is a reading of this window alone, so it stays. + expect(within(rail).getByText("LLM calls with a cost")).toBeTruthy() + }) + + it("turns the comparison off through the URL rather than through local state", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(screen.getByRole("button", { name: /compare prev/ })) + expect(onSearchChange).toHaveBeenCalledWith({ compare: false }) + }) + + it("writes the failing-only toggle to the URL", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(screen.getByRole("button", { name: "Failing only" })) + expect(onSearchChange).toHaveBeenCalledWith({ hasErrors: true }) + }) + + it("shows one chip per active filter and clears them all at once", () => { + const { onSearchChange } = renderView({ model: "claude-opus-5", environment: "production" }) + expect(screen.getAllByText("claude-opus-5").length).toBeGreaterThan(0) + fireEvent.click(screen.getByRole("button", { name: "Clear all" })) + expect(onSearchChange).toHaveBeenCalledWith({ + model: undefined, + agent: undefined, + service: undefined, + framework: undefined, + environment: undefined, + tool: undefined, + }) + }) + + it("removes one chip without touching the others", () => { + const { onSearchChange } = renderView({ model: "claude-opus-5" }) + fireEvent.click(screen.getByRole("button", { name: /Remove model filter/ })) + expect(onSearchChange).toHaveBeenCalledWith({ model: undefined }) + }) + + it("filters the whole page from a breakdown row", () => { + const { onSearchChange, data } = renderView({}) + const row = data.breakdowns.find((b) => b.dimension === "model")?.rows[0] + expect(row).toBeDefined() + fireEvent.click(within(sectionOf("Breakdowns")).getAllByText(row!.label)[0]) + expect(onSearchChange).toHaveBeenCalledWith({ model: row!.key }) + }) + + it("switches the breakdown table without touching the URL", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(within(sectionOf("Breakdowns")).getByRole("button", { name: /^tool/ })) + expect(screen.getByText("Share of calls")).toBeTruthy() + expect(onSearchChange).not.toHaveBeenCalled() + }) + + it("filters the page from a mover line", () => { + const { onSearchChange, data } = renderView({}) + const mover = data.movers.find((candidate) => candidate.key !== "") + expect(mover).toBeDefined() + const rail = screen.getByRole("heading", { name: "What changed" }).closest("aside")! + fireEvent.click(within(rail).getAllByText(mover!.label)[0]) + expect(onSearchChange).toHaveBeenCalledWith({ [mover!.dimension]: mover!.key }) + }) + + it("prints every service a top session touched, as the Sessions list does", () => { + const { fixture } = renderView({}) + const first = fixture.topSessions.cost[0] + const row = screen.getByText(first.sessionId).closest("div")! + expect(within(row).getByText(first.serviceNames.join(" · "))).toBeTruthy() + }) + + it("links each top session to its own detail page, with the session's bounds", () => { + const { fixture } = renderView({}) + const first = fixture.topSessions.cost[0] + const link = screen.getByText(first.sessionId).closest("a") + expect(link?.getAttribute("data-to")).toBe("/agent-sessions/$sessionId") + expect(link?.getAttribute("data-params")).toBe(JSON.stringify({ sessionId: first.sessionId })) + expect(link?.getAttribute("data-search")).toContain('"t"') + }) + + it("carries the board's filters into the Sessions list", () => { + renderView({ model: "claude-opus-5", hasErrors: true }) + const link = screen.getByText(/Open in Sessions/).closest("a") + expect(JSON.parse(link?.getAttribute("data-search") ?? "{}")).toMatchObject({ + models: ["claude-opus-5"], + hasErrors: true, + }) + }) + + it("switches the top-sessions tab without touching the URL", () => { + const { onSearchChange } = renderView({}) + fireEvent.click(screen.getByRole("button", { name: /longest/ })) + expect(onSearchChange).not.toHaveBeenCalled() + }) + + it("keeps the chips up and shows the empty block when the scope matches nothing", () => { + const fixture = buildOverviewFixture("healthy7d", NOW) + const data = buildAgentOverviewData({ + ...fixture.input, + current: { ...fixture.input.current, sessions: 0 }, + series: [], + compare: true, + }) + render( + , + ) + expect(screen.getByText("No agent sessions in this range")).toBeTruthy() + expect(screen.getAllByText("claude-opus-5").length).toBeGreaterThan(0) + expect(screen.queryByText("Top sessions")).toBeNull() + }) + + it("names the two tabs the page can be read under", () => { + renderView({}) + const nav = screen.getByRole("navigation", { name: "Agent sessions views" }) + expect(within(nav).getByText("Overview")).toBeTruthy() + expect(within(nav).getByText("Sessions")).toBeTruthy() + }) +}) + +/** + * A read that FAILED is not an empty window, and the page has to say which. + * Only the summary's failure takes the body — everything in it is made of that + * one read — and it keeps the chrome, which is the only way to change the + * window or the scope without leaving the page. + */ +describe("AgentOverviewView failures", () => { + it("keeps the header, the tabs and the toolbar when the summary read failed", () => { + const onRetry = vi.fn() + const fixture = buildOverviewFixture("healthy7d", NOW) + render( + , + ) + + expect(screen.getByRole("heading", { name: "Overview" })).toBeTruthy() + expect(screen.getByRole("navigation", { name: "Agent sessions views" })).toBeTruthy() + expect(screen.getByRole("combobox", { name: "model" })).toBeTruthy() + expect(screen.getByText("Failed to load the agent overview")).toBeTruthy() + // Nothing made of the summary is drawn beside it. + expect(screen.queryByText("Top sessions")).toBeNull() + + fireEvent.click(screen.getByRole("button", { name: "Try again" })) + expect(onRetry).toHaveBeenCalledTimes(1) + }) + + it("draws a failed breakdown in its own table rather than as an empty dimension", () => { + renderView({}, "regression24h", { breakdowns: { model: readFailure() } }) + const breakdowns = sectionOf("Breakdowns") + expect(within(breakdowns).getByText("Failed to load the model breakdown")).toBeTruthy() + expect(within(breakdowns).queryByText("No model activity in this range.")).toBeNull() + // The other five dimensions are unaffected, and their tabs still switch. + fireEvent.click(within(breakdowns).getByRole("button", { name: /^tool/ })) + expect(within(breakdowns).queryByText("Failed to load the model breakdown")).toBeNull() + expect(screen.getByText("Share of calls")).toBeTruthy() + }) + + it("draws a failed top-sessions read rather than saying no sessions match", () => { + renderView({}, "regression24h", { topSessions: readFailure() }) + const sessions = sectionOf("Top sessions") + expect(within(sessions).getByText("Failed to load the top sessions")).toBeTruthy() + expect(within(sessions).queryByText("No sessions match this scope.")).toBeNull() + }) + + it("replaces the model mix plot alone when its read failed", () => { + renderView({}, "regression24h", { modelMix: readFailure() }) + const cell = document.querySelector('[data-chart="modelMix"]')! + expect(within(cell as HTMLElement).getByRole("alert")).toBeTruthy() + // The other eight come from the summary and still plot. + expect(document.querySelectorAll("[data-plot]")).toHaveLength(8) + }) + + it("says the filter options are missing and leaves a set filter clearable", () => { + // A facets failure leaves every select with nothing to offer, which is + // what the note explains. + const fixture = buildOverviewFixture("healthy7d", NOW) + const data = buildAgentOverviewData({ ...fixture.input, compare: true }) + render( + , + ) + + expect(screen.getByText("Filter options unavailable")).toBeTruthy() + // A select with no options cannot be chosen from; the one holding the + // filter has to stay usable, or the filter cannot be removed here. + const model = screen.getByRole("combobox", { name: "model" }) as HTMLButtonElement + const agent = screen.getByRole("combobox", { name: "agent" }) as HTMLButtonElement + expect(model.disabled).toBe(false) + expect(agent.disabled).toBe(true) + }) +}) diff --git a/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx b/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx new file mode 100644 index 000000000..9a1135a82 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/agent-overview-view.tsx @@ -0,0 +1,244 @@ +import { useMemo, type ReactNode } from "react" + +import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" + +import { SquareSparkleIcon } from "@/components/icons" +import { QueryErrorState } from "@/components/common/query-error-state" +import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { AgentSessionsTabs } from "@/components/agent-sessions/tools/agent-sessions-tabs" +import type { TimeRangeSearch } from "@/components/time-range-picker/search" +import type { DetectedModel } from "@/hooks/use-detected-models" +import { + overviewScopeSummary, + type AgentOverviewData, + type OverviewMover, +} from "@/lib/agent-sessions/overview-analytics" +import { bucketWidthLabel } from "@/lib/agent-sessions/overview-buckets" +import { + activeOverviewFilters, + clearOverviewFilters, + compareEnabled, + sessionsLinkSearch, + toggleOverviewFilter, + type AgentOverviewSearch, + type OverviewDimension, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import type { OverviewTopSessionTab } from "@/lib/agent-sessions/use-agent-overview" + +import { OverviewBreakdowns } from "./overview-breakdowns" +import { OverviewFilterToolbar } from "./overview-filter-toolbar" +import { OverviewMetricStrip } from "./overview-metric-strip" +import { OverviewMoversRail } from "./overview-movers-rail" +import { OverviewScopeRow } from "./overview-scope-row" +import { OverviewTopSessions } from "./overview-top-sessions" +import { OverviewTrends } from "./overview-trends" + +/** + * The failures the page draws rather than swallows, one per read. + * + * A section that failed says so where its own numbers would have been, because + * an empty table and a table that could not be read are different facts about + * the window. Only the summary's failure takes the body: everything in it — the + * tiles, the nine headlines, the empty state — is made of that one read. + * + * `facets` is a flag and not an error: the selects lose their options and + * nothing else, so the toolbar says so in a line. + */ +export interface AgentOverviewErrors { + readonly summary?: unknown + readonly facets?: boolean + readonly breakdowns?: Partial> + readonly modelMix?: unknown + readonly topSessions?: unknown +} + +export interface AgentOverviewViewProps { + search: AgentOverviewSearch + /** Applied to the URL by the route. Keys set to `undefined` are cleared. */ + onSearchChange: (patch: Partial) => void + /** Absent only when the summary read failed; the host renders its own + * skeleton while that read is in flight. */ + data?: AgentOverviewData + errors?: AgentOverviewErrors + /** Re-runs every read on the page — what a failed summary offers. */ + onRetry?: () => void + facets: OverviewFacets + /** The active top-sessions tab's rows, and the tab itself — it lives with + * whoever issues that read rather than inside the table. */ + topSessions: ReadonlyArray + topSessionTab: OverviewTopSessionTab + onTopSessionTabChange: (tab: OverviewTopSessionTab) => void + /** Resolves a model to its vendor and display name; a warehouse read, so the + * page's owner does it and this tree stays presentational. */ + detectModel?: (model: string) => DetectedModel + /** Names the window and its comparison in the tiles, e.g. `7d`. */ + windowLabel: string + /** The window, carried by the tab strip's link to the Sessions list. */ + timeRange?: TimeRangeSearch + /** The time-range picker, or whatever the host wants beside the title. */ + headerControls?: ReactNode + /** Dim the data surfaces while a refetch is in flight. */ + waiting?: boolean +} + +/** + * The whole `/agent-sessions/overview` page below the layout chrome, over data + * that has already resolved. + * + * Presentational on purpose: the route hands it resolved values and the lab + * hands it fixtures, so the page can be looked at and reviewed without a + * warehouse behind it — `ai_trace_index` does not exist in the local Tinybird + * container. Every control writes a search param and nothing filters rows + * locally: the toolbar's predicates are server-side on every read, so the + * tiles, the grid and the tables always describe the same sessions. + * + * One column of full-bleed sections divided by hairlines, not a stack of cards: + * the page is one instrument, and every section is a different reading of the + * same scope. Reading order is the order the questions get asked: what am I + * looking at, over which sessions, narrowed to what, how much of it, how it + * moved and what moved most, grouped how, and finally which sessions. + */ +export function AgentOverviewView({ + search, + onSearchChange, + data, + errors, + onRetry, + facets, + topSessions, + topSessionTab, + onTopSessionTabChange, + detectModel, + windowLabel, + timeRange, + headerControls, + waiting, +}: AgentOverviewViewProps) { + const chips = useMemo(() => activeOverviewFilters(search), [search]) + const selectDimension = (dimension: OverviewDimension, key: string) => + onSearchChange(toggleOverviewFilter(search, dimension, key)) + + const note = + data === undefined + ? "" + : [ + compareEnabled(search) ? `previous ${windowLabel}` : `last ${windowLabel}`, + `${bucketWidthLabel(data.bucketSeconds)} buckets`, + ].join(" · ") + + return ( +
+
+
+

+ Overview +

+

+ Volume, cost, tokens, reliability and latency for every agent session — all on one + clock. +

+
+ {headerControls ? ( +
{headerControls}
+ ) : null} +
+ + + + + + {/* The whole board is the summary's, so its failure takes the body — + under the chrome, which is what the window and the scope are changed + from and is the only way back from here without leaving the page. */} + {data === undefined ? ( +
+ +
+ ) : ( + <> + {/* The chips stay up when the scope matches nothing: a reader looking + at an empty board needs to see what emptied it. */} + selectDimension(chip.dimension, chip.value)} + onClearAll={() => onSearchChange(clearOverviewFilters())} + /> + + {data.current.sessions === 0 ? ( + + + + + + No agent sessions in this range + + {chips.length === 0 + ? "Nothing your agents ran was recorded in this window. Widen the range, or check that the SDK is reporting." + : "Nothing in this window matches the scope above. Remove a filter to widen it."} + + + + ) : ( + <> + + + + selectDimension(mover.dimension, mover.key) + } + /> + } + /> + + + + + + )} + + )} +
+ ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx b/apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx new file mode 100644 index 000000000..0ad42f9ff --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-breakdowns.tsx @@ -0,0 +1,338 @@ +import { useState } from "react" + +import { formatErrorRate, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { cn } from "@maple/ui/lib/utils" + +import { ChevronRightIcon } from "@/components/icons" +import { QueryErrorState } from "@/components/common/query-error-state" +import { + deltaToneClass, + formatOverviewCount, + type OverviewBreakdown, + type OverviewBreakdownRow, + type OverviewDelta, + type OverviewModelMix, +} from "@/lib/agent-sessions/overview-analytics" +import { overviewModelMixColor } from "@/lib/agent-sessions/overview-chart-specs" +import { formatCost } from "@/lib/agent-sessions/session-summary" +import { + selectedDimensionValue, + type AgentOverviewSearch, + type OverviewDimension, +} from "@/lib/agent-sessions/overview-search" +import { vendorIcon } from "@/lib/agent-sessions/vendor-icon" + +export interface OverviewBreakdownsProps { + /** All six, in the dimensions' own order. */ + breakdowns: ReadonlyArray + /** The plotted bands, so a model's chip here is its band in the mix chart. */ + modelMix: OverviewModelMix + search: AgentOverviewSearch + /** Toggles that dimension's filter for the whole page. */ + onSelectRow: (dimension: OverviewDimension, key: string) => void + /** The read behind a dimension, where it failed — drawn in place of that + * table so a failed read is not read as an empty window. */ + errors?: Partial> + waiting?: boolean +} + +/** A lane: its header label, its width, and the width it sheds at. */ +interface Column { + readonly label: string + readonly className: string +} + +/** + * The columns a dimension can actually attribute — see `AiOverviewBreakdownRow`. + * + * A lane's width and shedding rule are written once and read by both the header + * and the cell, so a column cannot go missing from one at a width where the + * other still draws it. + */ +const USAGE_COLUMNS = [ + { label: "Share of cost", className: "w-[170px] @max-[900px]/table:hidden" }, + { label: "Sessions", className: "w-[84px] text-right" }, + { label: "LLM calls", className: "w-[92px] text-right @max-[700px]/table:hidden" }, + { label: "Tok / sess", className: "w-[92px] text-right @max-[820px]/table:hidden" }, + { label: "Cost", className: "w-[92px] text-right" }, + { label: "$ / sess", className: "w-[88px] text-right @max-[640px]/table:hidden" }, + { label: "Error rate", className: "w-[116px] text-right" }, + { label: "Δ prev", className: "w-[86px] text-right @max-[560px]/table:hidden" }, +] as const satisfies ReadonlyArray + +const TOOL_COLUMNS = [ + { label: "Share of calls", className: "w-[170px] @max-[900px]/table:hidden" }, + { label: "Sessions", className: "w-[84px] text-right" }, + { label: "Calls", className: "w-[92px] text-right" }, + { label: "Errors", className: "w-[92px] text-right @max-[700px]/table:hidden" }, + { label: "Error rate", className: "w-[116px] text-right" }, + { label: "Δ prev", className: "w-[86px] text-right @max-[560px]/table:hidden" }, +] as const satisfies ReadonlyArray + +/** + * The same window, grouped six ways. + * + * A row is not a link but a filter: clicking one narrows every reading on the + * page to that key, and clicking it again gives the page back. Rows OVERLAP — + * a session that used two models is a session under each — which the footer + * says out loud rather than leaving a reader to reconcile the columns. + */ +export function OverviewBreakdowns({ + breakdowns, + modelMix, + search, + onSelectRow, + errors, + waiting = false, +}: OverviewBreakdownsProps) { + const [active, setActive] = useState("model") + const breakdown = breakdowns.find((item) => item.dimension === active) + const isTool = active === "tool" + const columns: ReadonlyArray = isTool ? TOOL_COLUMNS : USAGE_COLUMNS + const selected = selectedDimensionValue(search, active) + const error = errors?.[active] + const rows = breakdown?.rows ?? [] + const worstRate = rows.reduce((max, row) => Math.max(max, row.errorRate), 0) + + return ( +
+
+
+

+ Breakdowns +

+ + click a row to filter the whole page + +
+
+ +
+ {breakdowns.map((item) => ( + + ))} +
+ + {error !== undefined ? ( +
+ +
+ ) : rows.length === 0 ? ( +

+ No {active} activity in this range. +

+ ) : ( +
+
+ {active} + {columns.map((column) => ( + + {column.label} + + ))} + +
+ + {rows.map((row) => ( + + ))} + +

+ {footerTotals(active, rows)} + + · + + + session counts overlap where a session used more than one {active} + {breakdown !== undefined && breakdown.totalKeys > rows.length + ? ` · +${breakdown.totalKeys - rows.length} more` + : ""} + +

+
+ )} +
+ ) +} + +const HEAD = "font-mono text-[10.5px] leading-[14px] tracking-[0.06em] text-muted-foreground/60 uppercase" +const NUM = "shrink-0 font-mono text-[12px] leading-4 tabular-nums" + +/** + * A model wears the colour of its band in the mix chart; a framework wears its + * vendor's mark. Nothing else gets a glyph — a lane of generic marks would + * indent every name without naming anything. + */ +function Glyph({ + dimension, + row, + modelMix, +}: { + dimension: OverviewDimension + row: OverviewBreakdownRow + modelMix: OverviewModelMix +}) { + if (dimension === "framework") { + const Icon = vendorIcon(row.key) + return + } + if (dimension !== "model") return null + const band = modelMix.models.indexOf(row.key) + return ( + + ) +} + +function UsageCells({ row, worstRate }: { row: OverviewBreakdownRow; worstRate: number }) { + return ( + <> + + + {formatOverviewCount(row.sessions)} + + + {formatOverviewCount(row.llmCalls)} + + + {formatNumber(row.tokensPerSession)} + + + {formatCost(row.cost)} + + + {formatCost(row.costPerSession)} + + + + + ) +} + +function ToolCells({ row, worstRate }: { row: OverviewBreakdownRow; worstRate: number }) { + return ( + <> + + + {formatOverviewCount(row.sessions)} + + + {formatOverviewCount(row.toolCalls)} + + + {formatOverviewCount(row.toolErrors)} + + + + + ) +} + +/** The row's share of what the table lists, drawn against the full width. */ +function ShareCell({ share, className }: { share: number; className: string }) { + return ( + + + 0 ? 2 : 0)}%` }} + /> + + + {formatPercent(share)} + + + ) +} + +/** + * The error rate, drawn against the WORST row rather than against 100%: a table + * where every model fails under 3% would otherwise be a column of invisible + * slivers, and ranking these rows against each other is the column's whole job. + * The tone is absolute, so the colour still says how bad 3% is. + */ +function RateCell({ rate, worst, className }: { rate: number; worst: number; className: string }) { + const tone = rate >= 0.1 ? "--severity-error" : rate >= 0.01 ? "--severity-warn" : "--severity-info" + return ( + + + 0 ? 4 : 0)}%`, + backgroundColor: `var(${tone})`, + }} + /> + + {formatErrorRate(rate)} + + ) +} + +/** A key with no previous window has no move to show, which is not a zero. */ +function DeltaCell({ delta, className }: { delta: OverviewDelta | null; className: string }) { + if (delta === null) { + return — + } + return {delta.text} +} + +/** What the listed rows add up to, as the closing line states it. */ +function footerTotals(dimension: OverviewDimension, rows: ReadonlyArray): string { + const plural = rows.length === 1 ? dimension : `${dimension}s` + if (dimension === "tool") { + const calls = rows.reduce((sum, row) => sum + row.toolCalls, 0) + return `${rows.length} ${plural} · ${formatOverviewCount(calls)} tool calls` + } + const calls = rows.reduce((sum, row) => sum + row.llmCalls, 0) + const cost = rows.reduce((sum, row) => sum + row.cost, 0) + return `${rows.length} ${plural} · ${formatOverviewCount(calls)} LLM calls · ${formatCost(cost)}` +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-delta.tsx b/apps/web/src/components/agent-sessions/overview/overview-delta.tsx new file mode 100644 index 000000000..d11693c42 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-delta.tsx @@ -0,0 +1,31 @@ +import { cn } from "@maple/ui/lib/utils" + +import { deltaToneClass, type OverviewDelta } from "@/lib/agent-sessions/overview-analytics" + +const ARROW = { up: "↑", down: "↓", flat: "→" } as const + +/** + * How a reading moved: an arrow for the direction, a colour for whether that + * was good news. + * + * Two signals, never one — colour alone is unreadable to a reader who cannot + * separate the greens from the reds, and an arrow alone cannot say that a + * falling cache-hit ratio is the bad kind of falling. The sign is dropped from + * the number because the arrow already carries it, and `↓ -4.9%` says it twice. + */ +export function DeltaReading({ delta, className }: { delta: OverviewDelta | null; className?: string }) { + if (delta === null) return null + return ( + + {ARROW[delta.direction]} + {delta.text.replace(/^[+-]/, "")} + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx new file mode 100644 index 000000000..0a918b7a1 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-filter-toolbar.tsx @@ -0,0 +1,170 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@maple/ui/components/ui/select" +import { cn } from "@maple/ui/lib/utils" + +import { CheckIcon } from "@/components/icons" +import { + OVERVIEW_DIMENSIONS, + compareEnabled, + failingOnly, + overviewFilterPatch, + selectedDimensionValue, + type AgentOverviewSearch, + type OverviewFacetOption, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import { formatOverviewCount } from "@/lib/agent-sessions/overview-analytics" + +/** Base UI selects need a real value for "no filter"; this never reaches the URL. */ +const ALL = "__all__" + +/** Every control on the row is the same pill, so a lit one reads as a narrowing. */ +const PILL = "inline-flex h-[30px] items-center rounded-md border px-2.5 font-mono text-xs" +const PILL_SET = "border-primary/40 bg-primary/10 text-primary" +const PILL_IDLE = "border-border bg-card text-muted-foreground hover:text-foreground" + +export interface OverviewFilterToolbarProps { + search: AgentOverviewSearch + facets: OverviewFacets + onSearchChange: (patch: Partial) => void + /** Names the comparison the toggle turns on, e.g. `7d`. */ + windowLabel: string + /** The facets read failed, so the selects have no options to offer. Said in + * a line rather than a panel: nothing else on the page depends on it. */ + facetsUnavailable?: boolean + waiting?: boolean +} + +/** + * Which sessions the board is about. + * + * Six dimensions, one value each: this page is read by narrowing to one thing + * at a time, and every control is the same 30px pill so a control drawn in the + * primary tint reads as "this is narrowing the page" at a glance. The two + * switches sit apart from the dimensions because they are not dimensions — one + * is a predicate on sessions, the other changes what the page compares against. + */ +export function OverviewFilterToolbar({ + search, + facets, + onSearchChange, + windowLabel, + facetsUnavailable = false, + waiting = false, +}: OverviewFilterToolbarProps) { + const failing = failingOnly(search) + const compare = compareEnabled(search) + return ( +
+
+ {OVERVIEW_DIMENSIONS.map((dimension) => ( + onSearchChange(overviewFilterPatch(dimension, value))} + /> + ))} + {facetsUnavailable ? ( + + Filter options unavailable + + ) : null} +
+ +
+ + + +
+
+ ) +} + +function FacetSelect({ + label, + value, + options, + onChange, +}: { + label: string + value: string | undefined + options: ReadonlyArray + onChange: (value: string | undefined) => void +}) { + const set = value !== undefined + return ( + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx b/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx new file mode 100644 index 000000000..6708fbb6a --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-metric-strip.tsx @@ -0,0 +1,78 @@ +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { cn } from "@maple/ui/lib/utils" + +import type { OverviewTile } from "@/lib/agent-sessions/overview-analytics" + +import { DeltaReading } from "./overview-delta" + +export interface OverviewMetricStripProps { + tiles: ReadonlyArray + waiting?: boolean +} + +/** + * Seven tiles, hairline-divided, in one band. + * + * `gap-px` over a border-coloured ground rather than `divide-x`: the strip + * folds to four columns and then to two, and a divide rule would leave a stray + * hairline at the page's own edge on every wrapped row. The last tile spans two + * columns where seven does not divide, so the ground never shows through as a + * missing tile. + */ +const STRIP = cn( + "grid grid-cols-2 gap-px border-b border-border bg-border", + "@min-[900px]/page:grid-cols-4 @min-[1200px]/page:grid-cols-7", + "[&>*:last-child]:col-span-2 @min-[1200px]/page:[&>*:last-child]:col-span-1", +) + +/** + * Seven readings of the window, left to right, and not a selector: nothing + * below the strip changes when one is read. The grid underneath already draws + * all nine series at once, so a tile that took over a chart would be a step + * backwards from what is already on screen. + */ +export function OverviewMetricStrip({ tiles, waiting = false }: OverviewMetricStripProps) { + return ( +
+ {tiles.map((tile) => ( +
+ + {tile.label} + + + + {tile.value} + + {tile.unit === undefined ? null : ( + + {tile.unit} + + )} + + + + {tile.sub} + +
+ ))} +
+ ) +} + +/** The strip's shape while the summary read is in flight. */ +export function OverviewMetricStripLoading() { + return ( +
+ {Array.from({ length: 7 }).map((_, index) => ( +
+ + + +
+ ))} +
+ ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx b/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx new file mode 100644 index 000000000..66a5db35f --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-movers-rail.tsx @@ -0,0 +1,153 @@ +import { formatPercent } from "@maple/ui/lib/format" +import { cn } from "@maple/ui/lib/utils" + +import { + OVERVIEW_MOVER_MIN_SESSIONS, + deltaToneClass, + type OverviewCoverage, + type OverviewMover, +} from "@/lib/agent-sessions/overview-analytics" + +export interface OverviewMoversRailProps { + movers: ReadonlyArray + coverage: OverviewCoverage + /** Off means there is no previous window to rank against — `movers` is empty + * then, and the rail says why rather than reading as "nothing moved". */ + compare: boolean + /** Names the window the moves are measured against, e.g. `7d`. */ + windowLabel: string + /** Applies that mover's dimension filter to the whole board. */ + onSelect: (mover: OverviewMover) => void +} + +/** + * What changed, ranked across every dimension at once. + * + * One line per key rather than per metric, and a magnitude bar measured against + * the worst move on the rail — the rail answers "where do I look first", which + * is an ordering question and not a measurement. The bar stays grey for the + * same reason: the ranking is the bar's whole message, and the delta beside it + * already says whether the move was bad news. + */ +export function OverviewMoversRail({ + movers, + coverage, + compare, + windowLabel, + onSelect, +}: OverviewMoversRailProps) { + const worst = movers.reduce((max, mover) => Math.max(max, mover.score), 0) + return ( + + ) +} + +/** The ranked lines themselves, and what the rail says when nothing ranks. */ +function MoverLines({ + movers, + worst, + onSelect, +}: { + movers: ReadonlyArray + worst: number + onSelect: (mover: OverviewMover) => void +}) { + return ( + <> +

+ Biggest movers across every breakdown. Click a line to filter the page. +

+ + {movers.length === 0 ? ( +

+ Nothing moved enough to rank. +

+ ) : ( +
    + {movers.map((mover, index) => ( +
  1. + +
    2. + ))} +
+ )} + +

+ Ranked by deviation; groups under {OVERVIEW_MOVER_MIN_SESSIONS} sessions in either window are + excluded. +

+ + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx b/apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx new file mode 100644 index 000000000..b714e7ef9 --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-scope-row.tsx @@ -0,0 +1,68 @@ +import { XmarkIcon } from "@/components/icons" + +import { OVERVIEW_DIMENSIONS, type OverviewFilterChip } from "@/lib/agent-sessions/overview-search" + +export interface OverviewScopeRowProps { + chips: ReadonlyArray + /** What the filters matched, in one sentence. */ + summary: string + onRemove: (chip: OverviewFilterChip) => void + onClearAll: () => void +} + +/** + * Everything narrowing the board, stated once — and, after it, everything that + * is not, so a reader can tell a scope of one dimension from a scope of five + * without counting chips. + * + * The chips stay visible when the filters match nothing: a reader who has + * narrowed to an empty set needs to see what they narrowed by, not an empty + * page with no explanation. + */ +export function OverviewScopeRow({ chips, summary, onRemove, onClearAll }: OverviewScopeRowProps) { + const narrowed = new Set(chips.map((chip) => chip.dimension)) + const open = OVERVIEW_DIMENSIONS.filter((dimension) => !narrowed.has(dimension)) + + return ( +
+ + Scope + + + {chips.map((chip) => ( + + ))} + + {chips.length === 0 ? null : ( + + )} + + {open.length === 0 ? null : ( + + {chips.length === 0 ? "" : "· "} + {open.map((dimension) => `all ${dimension}s`).join(" · ")} + + )} + + + {summary} + +
+ ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx b/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx new file mode 100644 index 000000000..e0f60605a --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-small-multiple.tsx @@ -0,0 +1,187 @@ +import { useMemo } from "react" +import { areaY, d3Curve, defineChart, lineY } from "@tanstack/charts" +import { scaleLinear } from "@tanstack/charts-scales/linear" +import { curveMonotoneX } from "d3-shape" + +import { ChartEmpty } from "@maple/ui/components/charts" +import { + PlotFrame, + PlotTooltipBody, + createTooltipFocusStore, + cursorTooltip, + dashedGridY, + focusCrosshair, + focusDot, + roundCapDasharray, + usePlotChromeColors, + type PlotTooltipSeries, +} from "@maple/ui/components/plot" + +import type { makeBucketAxis } from "@/components/infra/chart-utils" +import { LinkedCursorOverlay, linkedCursorChartProps } from "@/hooks/use-linked-cursor" +import { + OVERVIEW_TICK_PADDING, + overviewAxisTick, + type OverviewPlotRow, + type OverviewPlotSpec, +} from "@/lib/agent-sessions/overview-chart-specs" + +/** The plot box, x-axis labels included — the design's 86px svg over its ticks. */ +export const OVERVIEW_PLOT_HEIGHT = 104 +const STROKE_WIDTH = 1.5 +const GHOST_STROKE_WIDTH = 1.2 +/** A stack layer is read by its area, so it is nearly opaque. */ +const BAND_FILL_OPACITY = 0.85 +/** The p50–p95 region carries a line of its own colour and must stay behind it. */ +const SPREAD_FILL_OPACITY = 0.18 + +export interface OverviewSmallMultipleProps { + /** Names the chart to the linked cursor and to assistive tech. */ + chartId: string + title: string + spec: OverviewPlotSpec + /** Built once for the whole grid, so all nine agree on where an instant sits. */ + axis: ReturnType + /** The y-axis gutter, sized by `overviewAxisGutter` over every spec in the + * grid rather than this one alone, so the nine plots line up. */ + gutter: number +} + +/** + * One of the nine plots. + * + * Every chart on the board is this component over a different spec: the shapes + * differ (lines, a stack, a spread) but the chrome must not, because the grid is + * read across as much as down. + */ +export function OverviewSmallMultiple({ chartId, title, spec, axis, gutter }: OverviewSmallMultipleProps) { + const chromeColors = usePlotChromeColors() + const focusStore = useMemo(() => createTooltipFocusStore(), []) + + const tooltipSeries = useMemo[]>( + () => + spec.marks.map((mark) => ({ + label: mark.label, + color: mark.color, + dashed: mark.kind === "ghost", + // A stack layer's own reading is its THICKNESS; where it was drawn is + // that thickness plus everything under it, which is what the row + // highlight measures against the cursor. + value: (plotRow: OverviewPlotRow) => + mark.kind === "band" + ? numberAt(plotRow, mark.key) - numberAt(plotRow, mark.base) + : readNumber(plotRow, mark.key), + position: (plotRow: OverviewPlotRow) => readNumber(plotRow, mark.key), + format: spec.format, + })), + [spec], + ) + + const definition = useMemo(() => { + const at = (plotRow: OverviewPlotRow) => plotRow.date + const valueOf = (key: string) => (plotRow: OverviewPlotRow) => readNumber(plotRow, key) + const curve = d3Curve(curveMonotoneX) + const ghostDash = roundCapDasharray(3, 3, GHOST_STROKE_WIDTH) + const bands = spec.marks.filter((mark) => mark.kind === "band" || mark.kind === "spread") + const lines = spec.marks.filter((mark) => mark.kind === "line" || mark.kind === "ghost") + + return defineChart({ + marks: [ + dashedGridY(), + // Fill first, then the lines that are read over it. + ...bands.map((mark) => + areaY(spec.rows, { + id: `${mark.key}-band`, + x: at, + y: valueOf(mark.key), + // An explicit floor: these stacks are built in the spec, where the + // fallback band and the model tail are decided, not by a layout. + y1: valueOf(mark.base ?? mark.key), + fill: mark.color, + fillOpacity: mark.kind === "spread" ? SPREAD_FILL_OPACITY : BAND_FILL_OPACITY, + // `areaY` strokes the closed polygon, baseline included — the top + // edge is a `lineY` where a chart wants one. + stroke: "none", + curve, + }), + ), + ...lines.map((mark) => + lineY(spec.rows, { + id: mark.key, + x: at, + y: valueOf(mark.key), + stroke: mark.color, + strokeWidth: mark.kind === "ghost" ? GHOST_STROKE_WIDTH : STROKE_WIDTH, + strokeDasharray: mark.kind === "ghost" ? ghostDash : undefined, + curve, + }), + ), + ...lines.map((mark) => focusDot(spec.rows, at, valueOf(mark.key), mark.color, chromeColors)), + focusCrosshair(chromeColors), + ], + scales: { + x: axis.x, + y: { + scale: scaleLinear().domain([0, spec.yMax]), + axis: { + line: false, + ticks: { + size: 0, + padding: OVERVIEW_TICK_PADDING, + // Two labels, the extremes — nine charts of laddered ticks is a + // wall of digits, and the question here is shape. + values: [0, spec.yMax], + format: (value: number) => overviewAxisTick(spec, value), + }, + }, + }, + }, + // The top tick sits on the highest plotted value, so the margin is what + // keeps its label — and the peak under it — inside the frame. + margin: { left: gutter, right: 6, top: 8 }, + focus: "group-x", + focusRing: false, + tooltip: cursorTooltip(focusStore.anchor), + }) + }, [spec, axis, chromeColors, focusStore, gutter]) + + if (spec.rows.length === 0) { + return No data in this range. + } + + return ( +
+ ( + axis.heading(plotRow.bucket)} + /> + )} + /> + +
+ ) +} + +/** A row field as a plotted value; anything else is a gap, not a zero. */ +function readNumber(plotRow: OverviewPlotRow, key: string): number | null { + const value = plotRow[key] + return typeof value === "number" ? value : null +} + +/** The same read where a missing floor genuinely means the axis. */ +function numberAt(plotRow: OverviewPlotRow, key: string | undefined): number { + if (key === undefined) return 0 + const value = plotRow[key] + return typeof value === "number" ? value : 0 +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx b/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx new file mode 100644 index 000000000..462c590fa --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-top-sessions.tsx @@ -0,0 +1,269 @@ +import { Link } from "@tanstack/react-router" + +import { formatNumber } from "@maple/ui/lib/format" +import { formatSessionDuration } from "@maple/ui/lib/replay-format" +import { formatRelativeTimeOrDate } from "@maple/ui/lib/time-format" +import { cn } from "@maple/ui/lib/utils" + +import { ExternalLinkIcon } from "@/components/icons" +import { QueryErrorState } from "@/components/common/query-error-state" +import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { ModelLabel } from "@/components/agent-sessions/model-label" +import { unresolvedModel, type DetectedModel } from "@/hooks/use-detected-models" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" +import { formatOverviewCount } from "@/lib/agent-sessions/overview-analytics" +import { formatCost } from "@/lib/agent-sessions/session-summary" +import { sessionLinkWindow } from "@/lib/agent-sessions/session-window" +import type { AgentSessionsLinkSearch } from "@/lib/agent-sessions/overview-search" +import { + OVERVIEW_TOP_SESSION_TABS, + type OverviewTopSessionTab, +} from "@/lib/agent-sessions/use-agent-overview" + +const TAB_LABEL = { + cost: "most expensive", + duration: "longest", + errored: "errored", +} satisfies Record + +/** A lane: its header label, and the width and shedding rule its cells share. */ +const COLUMNS = [ + { label: "Agent", className: "w-[130px] @max-[760px]/table:hidden" }, + { label: "Model", className: "w-[150px] @max-[900px]/table:hidden" }, + { label: "Service", className: "w-[110px] @max-[1040px]/table:hidden" }, + { label: "Cost", className: "w-[78px] text-right" }, + { label: "Tokens", className: "w-[78px] text-right" }, + { label: "LLM", className: "w-[56px] text-right @max-[620px]/table:hidden" }, + { label: "Tools", className: "w-[56px] text-right @max-[620px]/table:hidden" }, + { label: "Errors", className: "w-[64px] text-right" }, + { label: "Duration", className: "w-[82px] text-right" }, + { label: "Started", className: "w-[96px] text-right @max-[680px]/table:hidden" }, +] as const + +export interface OverviewTopSessionsProps { + /** The active tab's rows — the only one of the three the page reads. */ + sessions: ReadonlyArray + active: OverviewTopSessionTab + /** The tab lives with whoever issues the read, so switching it issues one. */ + onActiveChange: (tab: OverviewTopSessionTab) => void + /** Sessions in the window that failed — what the Errored tab is a sample of. */ + erroredCount: number + /** The board's filters, as the Sessions list takes them. */ + sessionsSearch: AgentSessionsLinkSearch + /** The page's model detection, passed in because resolving a model is a read + * and this table is presentational. Absent, a model reads as its raw id. */ + detectModel?: (model: string) => DetectedModel + /** The list read's failure, where it failed — drawn in place of the table, + * so no rows is not read as no sessions. */ + error?: unknown + waiting?: boolean +} + +/** + * The concrete examples behind the trends above. + * + * Six rows of the sessions list itself, under the board's own filters: the same + * read the list page makes, ranked and cut to six, so these rows ARE that + * list's rows. The link out carries the filters and drops the ranking: the list + * is where the rest of them are. + */ +export function OverviewTopSessions({ + sessions: rows, + active, + onActiveChange, + erroredCount, + sessionsSearch, + detectModel = unresolvedModel, + error, + waiting = false, +}: OverviewTopSessionsProps) { + const { effectiveTimezone } = useTimezonePreference() + return ( +
+
+
+

+ Top sessions +

+ + the concrete examples behind the trends above + +
+ +
+ {OVERVIEW_TOP_SESSION_TABS.map((tab) => ( + + ))} + + Open in Sessions + + +
+
+ + {error !== undefined ? ( +
+ +
+ ) : rows.length === 0 ? ( +

+ No sessions match this scope. +

+ ) : ( +
+
+ Session + {COLUMNS.map((column) => ( + + {column.label} + + ))} + +
+ + {rows.map((row) => ( +
+ + {row.sessionId} + + + {row.firstAgentName || "—"} + + + + + + {row.serviceNames.join(" · ") || "—"} + + + {formatCost(row.cost)} + + {formatNumber(row.totalTokens)} + {formatOverviewCount(row.llmCalls)} + {formatOverviewCount(row.toolCalls)} + 0 + ? "text-[var(--severity-error)]" + : "text-muted-foreground/50" + } + > + {formatOverviewCount(row.errorSpanCount)} + + + {formatSessionDuration(row.durationMs)} + + + {/* The Sessions list's own reading: relative inside the week, + an absolute date once "23d ago" stops being the easier one. */} + {formatRelativeTimeOrDate(row.startTime, undefined, effectiveTimezone)} + + + + +
+ ))} + +

+ + {active === "errored" + ? `${rows.length} of ${formatOverviewCount(erroredCount)} errored sessions` + : `${rows.length} sessions`} + + + · + + + opening a session lands on its Trace view + +

+
+ )} +
+ ) +} + +const HEAD = "font-mono text-[10.5px] leading-[14px] tracking-[0.06em] text-muted-foreground/60 uppercase" + +/** The lane the Sessions list draws: the first model by name and mark, the rest + * as a count, and every raw id the row reported in the title. */ +function ModelCell({ + models, + detect, +}: { + models: ReadonlyArray + detect: (model: string) => DetectedModel +}) { + const first = models[0] + if (first === undefined) return <>— + return ( + + ) +} + +function Cell({ + className, + tone = "text-muted-foreground", + title, + children, +}: { + className: string + tone?: string + title?: string + children: React.ReactNode +}) { + return ( + + {children} + + ) +} diff --git a/apps/web/src/components/agent-sessions/overview/overview-trends.tsx b/apps/web/src/components/agent-sessions/overview/overview-trends.tsx new file mode 100644 index 000000000..c9b012c5e --- /dev/null +++ b/apps/web/src/components/agent-sessions/overview/overview-trends.tsx @@ -0,0 +1,318 @@ +import { useMemo, type ReactNode } from "react" + +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { cn } from "@maple/ui/lib/utils" + +import { QueryErrorState } from "@/components/common/query-error-state" +import { makeBucketAxis } from "@/components/infra/chart-utils" +import { useLinkedCursor } from "@/hooks/use-linked-cursor" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" +import { + OVERVIEW_CHARTS, + type OverviewChartId, + type OverviewChartSummary, + type OverviewModelMix, + type OverviewSeriesPoint, +} from "@/lib/agent-sessions/overview-analytics" +import { + buildOverviewPlotSpec, + overviewAxisGutter, + type OverviewPlotLegendItem, + type OverviewPlotSpec, +} from "@/lib/agent-sessions/overview-chart-specs" + +import { DeltaReading } from "./overview-delta" +import { OVERVIEW_PLOT_HEIGHT, OverviewSmallMultiple } from "./overview-small-multiple" + +export interface OverviewTrendsProps { + charts: ReadonlyArray + /** One point per bucket of the selected window, oldest first. */ + series: ReadonlyArray + /** The previous period, already shifted onto this axis. Empty with compare off. */ + previousSeries: ReadonlyArray + modelMix: OverviewModelMix + /** The mix read's failure, where it failed. Its own cell and not the whole + * grid: the other eight plots come from the summary and still hold. */ + modelMixError?: unknown + /** The bucket width and range, e.g. `6h buckets · previous 7 days`. */ + note: string + /** The "What changed" rail, placed beside the grid. */ + rail: ReactNode + waiting?: boolean +} + +/** + * Nine readings of one window at one bucket width, in one grid. + * + * Small multiples rather than one chart with a metric picker: the questions a + * regression raises are "did anything else move at the same instant", and that + * is a comparison across charts, not a sequence of them. One axis is built here + * for all nine — a shared bucket domain is what lets a rule drawn at the same + * fraction of every plot land on the same instant, which is the whole basis of + * the linked cursor below. + */ +export function OverviewTrends({ + charts, + series, + previousSeries, + modelMix, + modelMixError, + note, + rail, + waiting = false, +}: OverviewTrendsProps) { + const { effectiveTimezone } = useTimezonePreference() + const { containerProps } = useLinkedCursor(true) + + // The axis is the CURRENT window's buckets and nothing else: the previous + // period is drawn shifted onto this same grid, and a union with its own + // buckets would stretch the domain past the window the page is reading. + const axis = useMemo(() => { + const sorted = series.map((point) => point.bucket).toSorted((a, b) => a - b) + const base = makeBucketAxis( + sorted.map((ms) => new Date(ms).toISOString()), + effectiveTimezone, + ) + const spanMs = sorted.length < 2 ? 0 : sorted[sorted.length - 1]! - sorted[0]! + return { + ...base, + // The shared tick format is written for a full-width chart: "Sep 5, 12:00 + // AM" is most of a 280px plot, so one label survived thinning and the axis + // read as unlabelled. These carry the terse form the design uses. + x: { + ...base.x, + axis: { + ...base.x.axis, + ticks: { ...base.x.axis.ticks, format: terseTick(spanMs, effectiveTimezone) }, + }, + }, + } + }, [series, effectiveTimezone]) + + const specs = useMemo(() => { + const input = { series, previousSeries, modelMix } + return Object.fromEntries( + OVERVIEW_CHARTS.map((id) => [id, buildOverviewPlotSpec(id, input)]), + ) as Record + }, [series, previousSeries, modelMix]) + + // One gutter for all nine, so the plots line up column to column however wide + // this window's labels turn out to be. + const gutter = useMemo(() => overviewAxisGutter(Object.values(specs)), [specs]) + + return ( +
+
+
+

+ Trends +

+ + nine metrics, one clock — hover any chart to move the crosshair on all nine + +
+ {note} +
+ +
+
+ {charts.map((chart) => ( + + ))} +
+
+ {rail} +
+
+
+ ) +} + +const DAY_MS = 86_400_000 + +/** + * An x tick as one of these plots can afford to print it: a clock alone inside + * a day, a date alone on a midnight boundary, and the pair only where a tick + * lands mid-day in a window that spans several. + */ +function terseTick(spanMs: number, timeZone: string | undefined): (value: Date) => string { + const day = new Intl.DateTimeFormat(undefined, { timeZone, month: "short", day: "numeric" }) + const clock = new Intl.DateTimeFormat(undefined, { + timeZone, + hour: "2-digit", + minute: "2-digit", + hour12: false, + }) + return (value: Date) => { + const time = clock.format(value).replace(/^24:/, "00:") + if (spanMs <= DAY_MS) return time + return time === "00:00" ? day.format(value) : `${day.format(value)} ${time}` + } +} + +/** + * Hairlines between columns, never between rows — the grid is one instrument + * and a full lattice would read as nine cards. The stacked breakpoints keep + * that true at two and three columns and fall back to row rules at one, where + * there is no column to divide. + */ +const CHART_GRID = cn( + "grid grid-cols-1 @min-[700px]/page:grid-cols-2 @min-[1000px]/page:grid-cols-3", + "[&>figure]:border-b [&>figure]:border-border [&>figure:last-child]:border-b-0", + "@min-[700px]/page:[&>figure]:border-r @min-[700px]/page:[&>figure]:border-b-0", + "@min-[700px]/page:[&>figure:nth-child(2n)]:border-r-0", + "@min-[1000px]/page:[&>figure:nth-child(2n)]:border-r", + "@min-[1000px]/page:[&>figure:nth-child(3n)]:border-r-0", + "@min-[700px]/page:[&>figure:last-child]:border-r-0", +) + +/** + * One small multiple: what it measures, where it stands now, how it moved, what + * the marks mean, and then the marks. + */ +function ChartCell({ + chart, + spec, + axis, + gutter, + error, +}: { + chart: OverviewChartSummary + spec: OverviewPlotSpec + axis: ReturnType + gutter: number + /** This chart's own read failed — the plot is replaced, the cell is not. */ + error?: unknown +}) { + return ( +
+
+ + + {chart.title} + + + {chart.value} + + + + + {chart.unit} + + + +
+ + {error === undefined ? ( +
+ {spec.legend.map((item) => ( + + ))} + {spec.legendMore === 0 ? null : ( + + +{spec.legendMore} + + )} +
+ ) : null} +
+ {error === undefined ? ( + + ) : ( + // At least the plot's own height, so a failed read does not + // collapse the cell out from under the grid's alignment. +
+ +
+ )} +
+
+ ) +} + +function LegendItem({ item }: { item: OverviewPlotLegendItem }) { + return ( + + + + {item.label} + + + ) +} + +function Swatch({ item }: { item: OverviewPlotLegendItem }) { + if (item.kind === "ghost") { + return ( + + ) + } + if (item.kind === "line") { + return ( + + ) + } + return ( + + ) +} + +/** The grid's shape while the summary read is in flight — nine cells, not one box. */ +export function OverviewTrendsLoading() { + return ( +
+
+ +
+
+ {OVERVIEW_CHARTS.map((id) => ( +
+ + + +
+ ))} +
+
+ ) +} diff --git a/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx b/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx new file mode 100644 index 000000000..b2fe41ec1 --- /dev/null +++ b/apps/web/src/components/agent-sessions/tools/agent-sessions-tabs.tsx @@ -0,0 +1,90 @@ +import { Link } from "@tanstack/react-router" + +import { cn } from "@maple/ui/lib/utils" + +import { ChartBarTrendUpIcon, LayersIcon } from "@/components/icons" +import { pickTimeRangeSearch, type TimeRangeSearch } from "@/components/time-range-picker/search" + +/** The two readings of the same spans: the whole population, or one session at a time. */ +export type AgentSessionsTab = "overview" | "sessions" + +/** + * The tab strip both Agent Sessions pages carry. + * + * Real links rather than a `Tabs` widget, because the two tabs are two routes: + * middle-click and Copy link have to work, and the browser's own Back is what + * undoes the switch. Only the window travels between them — the overview's + * dimension filters mean nothing to a list that pages one session at a time. + * + * The tab already showing is the exception: it navigates to where it already + * is, so it MERGES rather than replaces. A `search` object replaces the whole + * search, and clicking the tab you are on is not how anyone asks for their + * filters to be cleared. + * + * The Sessions list has no time picker (it is fixed to a rolling week), so a + * jump from there carries no window and this page falls back to its own default. + */ +export function AgentSessionsTabs({ + active, + search, + className, +}: { + active: AgentSessionsTab + /** The current window, carried across. Absent from the Sessions list, which has none. */ + search?: TimeRangeSearch + className?: string +}) { + const window = search === undefined ? {} : pickTimeRangeSearch(search) + return ( + + ) +} + +function TabLink({ + to, + search, + active, + icon, + children, +}: { + to: "/agent-sessions" | "/agent-sessions/overview" + /** Only the window travels; the Sessions list simply drops what it does not validate. */ + search: TimeRangeSearch + active: boolean + icon: React.ReactNode + children: React.ReactNode +}) { + return ( + ) => ({ ...prev, ...search }) : search} + aria-current={active ? "page" : undefined} + className={cn( + "flex h-9 items-center gap-[7px] border-b-2 px-3 font-mono text-[12.5px] transition-colors first:pl-0.5", + active + ? "border-primary font-medium text-foreground [&_svg]:text-primary" + : "border-transparent text-muted-foreground hover:text-foreground [&_svg]:text-muted-foreground", + )} + > + {icon} + {children} + + ) +} diff --git a/apps/web/src/hooks/use-detected-models.ts b/apps/web/src/hooks/use-detected-models.ts index d7efbd20f..a9fa2449a 100644 --- a/apps/web/src/hooks/use-detected-models.ts +++ b/apps/web/src/hooks/use-detected-models.ts @@ -19,7 +19,8 @@ export interface DetectedModel { readonly family: string | null } -const unresolved = (model: string): DetectedModel => ({ +/** What a model reads as before — or without — a detection read. */ +export const unresolvedModel = (model: string): DetectedModel => ({ model, displayName: shortTarget(model), vendorSlug: null, @@ -67,6 +68,6 @@ export function useDetectedModels(models: ReadonlyArray): (model: string .onSuccess((value: ReadonlyArray) => value) .orElse(() => [] as ReadonlyArray) const byModel = new Map(detected.map((entry) => [entry.model, entry])) - return (model: string) => byModel.get(model.trim()) ?? unresolved(model) + return (model: string) => byModel.get(model.trim()) ?? unresolvedModel(model) }, [result]) } diff --git a/apps/web/src/lab/agent-overview-fixture.ts b/apps/web/src/lab/agent-overview-fixture.ts new file mode 100644 index 000000000..28058d4df --- /dev/null +++ b/apps/web/src/lab/agent-overview-fixture.ts @@ -0,0 +1,545 @@ +// Two boards' worth of synthetic agent traffic, for `/lab/agent-overview` and +// for the view's own test. +// +// `ai_trace_index` does not exist in the local Tinybird container, so the real +// page has nothing to draw locally; this is where its layout gets looked at. +// +// Every figure is built from PER-SESSION rates rather than typed in as totals, +// so the counts, the ratios and the rates in a bucket can never contradict each +// other the way hand-written fixture numbers do. The scenarios are the two the +// design was drawn for: a healthy week, and a day with a step change at 14:00 +// UTC that the movers rail is supposed to find. +// +// The filters are NOT applied to these numbers — the lab writes them to local +// state so every control, chip and row highlight works, and the readings stay +// put so a layout change is the only thing that moves on screen. + +import { formatWarehouseDateTime } from "@maple/query-engine" + +import type { AgentSessionRow } from "@/components/agent-sessions/agent-sessions-list" +import { + EMPTY_OVERVIEW_MEASURES, + type AgentOverviewInput, + type OverviewBreakdownEntry, + type OverviewMeasurePoint, + type OverviewMeasures, + type OverviewModelMixRow, +} from "@/lib/agent-sessions/overview-analytics" +import { + OVERVIEW_DIMENSIONS, + type OverviewDimension, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import type { OverviewTopSessionTab } from "@/lib/agent-sessions/use-agent-overview" + +export const OVERVIEW_SCENARIOS = ["healthy7d", "regression24h"] as const +export type OverviewScenario = (typeof OVERVIEW_SCENARIOS)[number] + +export interface OverviewFixture { + readonly scenario: OverviewScenario + readonly windowLabel: string + readonly window: { readonly startTime: string; readonly endTime: string } + /** Ready for `buildAgentOverviewData`, minus the `compare` the lab owns. */ + readonly input: Omit + readonly facets: OverviewFacets + readonly topSessions: Record> +} + +/* ------------------------------------------------------------------------------------------------- + * Shapes + * -----------------------------------------------------------------------------------------------*/ + +/** One bucket's behaviour, as a reader would describe it. */ +interface SessionRates { + sessions: number + /** Sessions with at least one failed span, 0–1. */ + errorRate: number + llmPerSession: number + /** Model-call SPANS per netted call — gateway mirrors and wrapper roll-ups. */ + spanFactor: number + llmErrorRate: number + toolsPerSession: number + toolErrorRate: number + costPerSession: number + tokensPerSession: number + /** Cache reads over everything that could have been a prompt read, 0–1. */ + cacheShare: number + pricedShare: number + p50Ms: number + p95Ms: number +} + +const HEALTHY: SessionRates = { + sessions: 114, + errorRate: 0.024, + llmPerSession: 8.4, + spanFactor: 1.18, + llmErrorRate: 0.019, + toolsPerSession: 6.3, + toolErrorRate: 0.031, + costPerSession: 0.264, + tokensPerSession: 67_400, + cacheShare: 0.71, + pricedShare: 0.94, + p50Ms: 42_000, + p95Ms: 96_000, +} + +/** The step the investigating board is drawn around. Sessions barely move. */ +const REGRESSED: SessionRates = { + ...HEALTHY, + errorRate: 0.26, + llmErrorRate: 0.161, + toolsPerSession: 15.8, + toolErrorRate: 0.19, + costPerSession: 0.378, + tokensPerSession: 118_000, + cacheShare: 0.12, + p95Ms: 227_000, +} + +/** A deterministic wobble, so a board looks like traffic and not like a ruler. */ +const wobble = (index: number, amplitude: number): number => + 1 + amplitude * Math.sin(index * 1.7) + (amplitude / 2) * Math.sin(index * 0.53) + +const scaleRates = (rates: SessionRates, index: number): SessionRates => ({ + ...rates, + sessions: Math.round(rates.sessions * wobble(index, 0.18)), + costPerSession: rates.costPerSession * wobble(index, 0.09), + tokensPerSession: rates.tokensPerSession * wobble(index, 0.07), + toolsPerSession: rates.toolsPerSession * wobble(index, 0.08), + errorRate: rates.errorRate * wobble(index, 0.22), + llmErrorRate: rates.llmErrorRate * wobble(index, 0.2), + p95Ms: rates.p95Ms * wobble(index, 0.11), +}) + +/** + * One bucket's rates, as the API would report them. + * + * The token split is disjoint and adds to the total, and `cacheShare` is + * exactly the ratio the cache-hit chart divides — the numbers agree because + * they come from one place. + */ +function measuresOf(rates: SessionRates): OverviewMeasures { + const sessions = Math.max(0, Math.round(rates.sessions)) + const llmCalls = Math.round(sessions * rates.llmPerSession) + const llmCallSpans = Math.round(llmCalls * rates.spanFactor) + const toolCalls = Math.round(sessions * rates.toolsPerSession) + const tokens = Math.round(sessions * rates.tokensPerSession) + const prompt = tokens * 0.78 + return { + ...EMPTY_OVERVIEW_MEASURES, + sessions, + erroredSessions: Math.round(sessions * rates.errorRate), + llmCalls, + llmCallSpans, + erroredLlmCalls: Math.round(llmCallSpans * rates.llmErrorRate), + toolCalls, + erroredToolCalls: Math.round(toolCalls * rates.toolErrorRate), + cost: Number((sessions * rates.costPerSession).toFixed(2)), + pricedLlmCalls: Math.round(llmCalls * rates.pricedShare), + tokens, + inputTokens: Math.round(prompt * (1 - rates.cacheShare)), + cacheReadTokens: Math.round(prompt * rates.cacheShare), + cacheWriteTokens: Math.round(tokens * 0.04), + outputTokens: Math.round(tokens * 0.12), + reasoningTokens: Math.round(tokens * 0.06), + sessionDurationP50Ms: rates.p50Ms, + sessionDurationP95Ms: rates.p95Ms, + } +} + +/** Counts sum; quantiles do not, so the window's own are passed in. */ +function foldMeasures( + points: ReadonlyArray, + quantiles: Pick, +): OverviewMeasures { + const sum = points.reduce( + (total, point) => ({ + ...total, + sessions: total.sessions + point.sessions, + erroredSessions: total.erroredSessions + point.erroredSessions, + llmCalls: total.llmCalls + point.llmCalls, + llmCallSpans: total.llmCallSpans + point.llmCallSpans, + erroredLlmCalls: total.erroredLlmCalls + point.erroredLlmCalls, + toolCalls: total.toolCalls + point.toolCalls, + erroredToolCalls: total.erroredToolCalls + point.erroredToolCalls, + cost: total.cost + point.cost, + pricedLlmCalls: total.pricedLlmCalls + point.pricedLlmCalls, + tokens: total.tokens + point.tokens, + inputTokens: total.inputTokens + point.inputTokens, + cacheReadTokens: total.cacheReadTokens + point.cacheReadTokens, + cacheWriteTokens: total.cacheWriteTokens + point.cacheWriteTokens, + outputTokens: total.outputTokens + point.outputTokens, + reasoningTokens: total.reasoningTokens + point.reasoningTokens, + }), + EMPTY_OVERVIEW_MEASURES, + ) + return { ...sum, cost: Number(sum.cost.toFixed(2)), ...quantiles } +} + +/* ------------------------------------------------------------------------------------------------- + * Breakdowns + * -----------------------------------------------------------------------------------------------*/ + +/** One key's story: its share of the window, and what it did differently. */ +interface KeyStory { + key: string + share: number + current?: Partial + previous?: Partial +} + +const stories = (...entries: ReadonlyArray) => entries + +const STORIES = { + model: stories( + { + key: "claude-opus-5", + share: 0.42, + // The regression the rail is supposed to put first. + current: { llmErrorRate: 0.161, costPerSession: 0.41 }, + previous: { llmErrorRate: 0.019, costPerSession: 0.29 }, + }, + { key: "gpt-5.5", share: 0.23 }, + { key: "claude-sonnet-5", share: 0.16, current: { costPerSession: 0.09 } }, + { key: "gemini-3-pro", share: 0.11, current: { tokensPerSession: 94_000 } }, + { key: "gpt-5.6", share: 0.05 }, + { key: "llama-4-70b", share: 0.03, current: { costPerSession: 0.004 } }, + ), + agent: stories( + { + key: "release-captain", + share: 0.31, + current: { toolsPerSession: 15.8, errorRate: 0.24 }, + previous: { toolsPerSession: 6.1, errorRate: 0.022 }, + }, + { key: "code-reviewer", share: 0.27 }, + { key: "docs-writer", share: 0.18, current: { tokensPerSession: 122_000 } }, + { key: "triage-bot", share: 0.14 }, + { key: "", share: 0.1 }, + ), + service: stories( + { key: "api", share: 0.46, current: { p95Ms: 188_000 }, previous: { p95Ms: 94_000 } }, + { key: "worker", share: 0.29 }, + { key: "cli", share: 0.17 }, + { key: "landing", share: 0.08 }, + ), + framework: stories( + { key: "eve", share: 0.58 }, + { key: "openrouter", share: 0.24, current: { costPerSession: 0.39 } }, + { key: "langchain", share: 0.13 }, + { key: "vercel-ai", share: 0.05 }, + ), + environment: stories( + { + key: "production", + share: 0.64, + current: { errorRate: 0.19 }, + previous: { errorRate: 0.021 }, + }, + { key: "staging", share: 0.26 }, + { key: "development", share: 0.1 }, + ), + tool: stories( + { + key: "run_tests", + share: 0.34, + current: { toolErrorRate: 0.28, toolsPerSession: 9.4 }, + previous: { toolErrorRate: 0.04, toolsPerSession: 3.6 }, + }, + { key: "read_file", share: 0.26 }, + { key: "search_code", share: 0.19 }, + { key: "apply_patch", share: 0.13, current: { toolErrorRate: 0.09 } }, + { key: "web_fetch", share: 0.08 }, + ), +} satisfies Record> + +/** + * One dimension's rows. + * + * The stories above are the REGRESSED board's; a healthy week gets a small + * deterministic drift per key instead, so its movers rail has the handful of + * modest moves a healthy week actually has rather than a copy of the outage. + */ +function breakdownEntries( + dimension: OverviewDimension, + current: SessionRates, + previous: SessionRates, + regressed: boolean, +): ReadonlyArray { + // Seeded per dimension as well as per row, so six tables do not print six + // copies of one key's drift and the rail ranks six different things. + const seed = OVERVIEW_DIMENSIONS.indexOf(dimension) * 7 + const drift = (rates: SessionRates, at: number): Partial => ({ + costPerSession: rates.costPerSession * wobble(seed + at, 0.16), + tokensPerSession: rates.tokensPerSession * wobble(seed + at + 2, 0.24), + errorRate: rates.errorRate * wobble(seed + at + 1, 0.3), + toolsPerSession: rates.toolsPerSession * wobble(seed + at + 3, 0.18), + }) + return STORIES[dimension].map((story, index) => ({ + key: story.key, + current: measuresOf({ + ...current, + sessions: current.sessions * story.share, + ...drift(current, index * 2), + ...(regressed ? story.current : undefined), + }), + previous: measuresOf({ + ...previous, + sessions: previous.sessions * story.share, + ...drift(previous, index * 2 + 11), + ...(regressed ? story.previous : undefined), + }), + })) +} + +/* ------------------------------------------------------------------------------------------------- + * Model mix + * -----------------------------------------------------------------------------------------------*/ + +/** Seven models, so the sixth and the seventh fold into the `other` band. */ +const MIX_MODELS = [ + { model: "claude-opus-5", base: 0.34, regressed: 0.62 }, + { model: "gpt-5.5", base: 0.24, regressed: 0.14 }, + { model: "claude-sonnet-5", base: 0.16, regressed: 0.09 }, + { model: "gemini-3-pro", base: 0.12, regressed: 0.07 }, + { model: "gpt-5.6", base: 0.07, regressed: 0.04 }, + { model: "llama-4-70b", base: 0.04, regressed: 0.02 }, + { model: "mistral-large-3", base: 0.03, regressed: 0.02 }, +] as const + +function modelMixRows( + buckets: ReadonlyArray<{ bucket: number; spans: number; regressed: boolean }>, +): ReadonlyArray { + return buckets.flatMap(({ bucket, spans, regressed }, index) => + MIX_MODELS.map((model) => ({ + bucket, + model: model.model, + llmCallSpans: Math.max( + 1, + Math.round(spans * (regressed ? model.regressed : model.base) * wobble(index, 0.06)), + ), + })), + ) +} + +/* ------------------------------------------------------------------------------------------------- + * Top sessions + * -----------------------------------------------------------------------------------------------*/ + +const SESSION_SEEDS = [ + { agent: "release-captain", model: "claude-opus-5", service: "api", vendor: "eve" }, + { agent: "code-reviewer", model: "gpt-5.5", service: "api", vendor: "openrouter" }, + { agent: "docs-writer", model: "gemini-3-pro", service: "worker", vendor: "eve" }, + { agent: "triage-bot", model: "claude-sonnet-5", service: "worker", vendor: "langchain" }, + { agent: "release-captain", model: "claude-opus-5", service: "cli", vendor: "eve" }, + { agent: "code-reviewer", model: "gpt-5.6", service: "api", vendor: "vercel-ai" }, +] as const + +function topSessions( + tab: OverviewTopSessionTab, + endMs: number, + rates: SessionRates, +): ReadonlyArray { + return SESSION_SEEDS.map((seed, index) => { + const rank = SESSION_SEEDS.length - index + const startMs = endMs - (index + 1) * 37 * 60_000 + const durationMs = + tab === "duration" ? rates.p95Ms * (1.6 + index * 0.2) : rates.p50Ms * (1 + index * 0.1) + const errors = tab === "errored" ? rank * 3 : index === 0 ? 2 : 0 + const llmCalls = Math.round(rates.llmPerSession * (tab === "cost" ? rank * 1.6 : 1.2)) + return { + sessionId: `${seed.agent}-${(2261 + index * 17).toString(16)}`, + vendorId: seed.vendor, + traceCount: 1 + (index % 3), + spanCount: 40 + index * 11, + errorSpanCount: errors, + toolErrorCount: Math.round(errors * 0.6), + turnErrorCount: errors - Math.round(errors * 0.6), + serviceNames: [seed.service], + models: [seed.model], + agentNames: [seed.agent], + firstAgentName: seed.agent, + llmCalls, + toolCalls: Math.round(rates.toolsPerSession * (tab === "cost" ? rank : 1.4)), + totalTokens: Math.round(rates.tokensPerSession * (tab === "cost" ? rank * 1.4 : 1.1)), + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cost: Number((rates.costPerSession * (tab === "cost" ? rank * 2.4 : 1.3)).toFixed(2)), + startTime: formatWarehouseDateTime(startMs), + endTime: formatWarehouseDateTime(startMs + durationMs), + durationMs: Math.round(durationMs), + hasDetails: true, + } + }) +} + +/* ------------------------------------------------------------------------------------------------- + * The scenarios + * -----------------------------------------------------------------------------------------------*/ + +const HOUR_MS = 60 * 60_000 +const MINUTE_MS = 60_000 +/** The grid the time picker floors a preset's endpoint to at this width. */ +const SNAP_MS = 15 * MINUTE_MS + +/** The hour the investigating board's step happens, in UTC so a board looks the + * same wherever it is opened. */ +export const OVERVIEW_REGRESSION_HOUR_UTC = 14 + +interface ScenarioSpec { + readonly windowLabel: string + readonly bucketSeconds: number + /** + * The window's own length — deliberately NOT a whole number of buckets. + * + * That is what the page actually gets: "7d" is calendar-aligned, so it runs + * from a midnight to a "now" floored to the quarter hour, and the ghost + * series has to land on the bucket grid anyway. + */ + readonly windowMs: number + readonly current: SessionRates + readonly previous: SessionRates + /** True where the bucket is past the step. Healthy weeks have none. */ + readonly regressedAt: (bucketMs: number) => boolean +} + +const SPECS = { + healthy7d: { + windowLabel: "7d", + bucketSeconds: 6 * 3_600, + windowMs: 6 * 24 * HOUR_MS + 14 * HOUR_MS + 15 * MINUTE_MS, + current: HEALTHY, + previous: { ...HEALTHY, sessions: 121, costPerSession: 0.276, errorRate: 0.027 }, + regressedAt: (_bucketMs: number) => false, + }, + regression24h: { + windowLabel: "24h", + bucketSeconds: 3_600, + windowMs: 23 * HOUR_MS + 45 * MINUTE_MS, + current: { ...HEALTHY, sessions: 52 }, + previous: { ...HEALTHY, sessions: 51 }, + regressedAt: (bucketMs: number) => new Date(bucketMs).getUTCHours() >= OVERVIEW_REGRESSION_HOUR_UTC, + }, +} satisfies Record + +/** + * One board's worth of data, from one frozen timestamp. + * + * The window ends where the picker would snap it and is a ragged number of + * buckets long, exactly as the page's own default is; both windows are then cut + * into epoch-aligned bucket starts, the way the warehouse cuts them, so the lab + * draws the ghost series the real alignment has to produce. + */ +export function buildOverviewFixture(scenario: OverviewScenario, nowMs: number): OverviewFixture { + const spec = SPECS[scenario] + const bucketMs = spec.bucketSeconds * 1_000 + const endMs = Math.floor(nowMs / SNAP_MS) * SNAP_MS + const startMs = endMs - spec.windowMs + const bucketStart = (ms: number) => Math.floor(ms / bucketMs) * bucketMs + const firstBucket = bucketStart(startMs) + const previousFirstBucket = bucketStart(startMs - spec.windowMs) + const bucketCount = Math.ceil((endMs - firstBucket) / bucketMs) + + const buckets = Array.from({ length: bucketCount }, (_, index) => { + const bucket = firstBucket + index * bucketMs + const regressed = spec.regressedAt(bucket) + return { bucket, index, regressed } + }) + + const series: ReadonlyArray = buckets.map(({ bucket, index, regressed }) => ({ + bucket, + ...measuresOf( + scaleRates(regressed ? { ...REGRESSED, sessions: spec.current.sessions } : spec.current, index), + ), + })) + const previousSeries: ReadonlyArray = buckets.map(({ index }) => ({ + bucket: previousFirstBucket + index * bucketMs, + ...measuresOf(scaleRates(spec.previous, index + 3)), + })) + + // The regressed scenario's window mixes both shapes, so the tiles read the + // whole window while the grid shows where it turned. + const regressedShare = buckets.filter((b) => b.regressed).length / bucketCount + const blend = (healthy: number, bad: number) => healthy * (1 - regressedShare) + bad * regressedShare + const currentRates: SessionRates = { + ...spec.current, + errorRate: blend(spec.current.errorRate, REGRESSED.errorRate), + llmErrorRate: blend(spec.current.llmErrorRate, REGRESSED.llmErrorRate), + toolsPerSession: blend(spec.current.toolsPerSession, REGRESSED.toolsPerSession), + toolErrorRate: blend(spec.current.toolErrorRate, REGRESSED.toolErrorRate), + costPerSession: blend(spec.current.costPerSession, REGRESSED.costPerSession), + cacheShare: blend(spec.current.cacheShare, REGRESSED.cacheShare), + p95Ms: blend(spec.current.p95Ms, REGRESSED.p95Ms), + } + + const current = foldMeasures(series, { + sessionDurationP50Ms: currentRates.p50Ms, + sessionDurationP95Ms: currentRates.p95Ms, + }) + const previous = foldMeasures(previousSeries, { + sessionDurationP50Ms: spec.previous.p50Ms, + sessionDurationP95Ms: spec.previous.p95Ms, + }) + + // Breakdown rows are measured over the WINDOW, like the tiles above them — + // a table summing to a fraction of the strip would read as a bug. + const breakdownCurrent: SessionRates = { ...currentRates, sessions: current.sessions } + const breakdownPrevious: SessionRates = { ...spec.previous, sessions: previous.sessions } + const breakdowns = OVERVIEW_DIMENSIONS.map((dimension) => { + const entries = breakdownEntries(dimension, breakdownCurrent, breakdownPrevious, regressedShare > 0) + return { dimension, entries, totalKeys: entries.length + (dimension === "tool" ? 9 : 4) } + }) + + const facetsFor = (dimension: OverviewDimension) => + STORIES[dimension] + .filter((story) => story.key !== "") + .map((story) => ({ + name: story.key, + count: Math.round(current.sessions * story.share), + })) + const facets: OverviewFacets = { + model: facetsFor("model"), + agent: facetsFor("agent"), + service: facetsFor("service"), + framework: facetsFor("framework"), + environment: facetsFor("environment"), + tool: facetsFor("tool"), + } + + return { + scenario, + windowLabel: spec.windowLabel, + window: { + startTime: formatWarehouseDateTime(startMs), + endTime: formatWarehouseDateTime(endMs), + }, + input: { + current, + previous, + series, + previousSeries, + modelMix: modelMixRows( + buckets.map(({ bucket, index, regressed }) => ({ + bucket, + spans: series[index].llmCallSpans, + regressed, + })), + ), + breakdowns, + bucketSeconds: spec.bucketSeconds, + windowMs: { startMs, endMs }, + windowLabel: spec.windowLabel, + }, + facets, + topSessions: { + cost: topSessions("cost", endMs, currentRates), + duration: topSessions("duration", endMs, currentRates), + errored: topSessions("errored", endMs, currentRates), + }, + } +} diff --git a/apps/web/src/lab/agent-overview-lab.tsx b/apps/web/src/lab/agent-overview-lab.tsx new file mode 100644 index 000000000..e790f2564 --- /dev/null +++ b/apps/web/src/lab/agent-overview-lab.tsx @@ -0,0 +1,110 @@ +import { useMemo, useState } from "react" + +import { AgentOverviewView } from "@/components/agent-sessions/overview/agent-overview-view" +import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" +import { compareEnabled, type AgentOverviewSearch } from "@/lib/agent-sessions/overview-search" +import type { OverviewTopSessionTab } from "@/lib/agent-sessions/use-agent-overview" + +import { OVERVIEW_SCENARIOS, buildOverviewFixture, type OverviewScenario } from "./agent-overview-fixture" + +/** + * The overview board without a warehouse behind it. + * + * The page is the real one — the route mounts this same view — over two + * synthetic boards: a healthy week, and a day whose 14:00 UTC step is what the + * movers rail and the error charts are for. The URL is stood in for by local + * state, so every control works: the selects and the toggles narrow the scope + * row, a breakdown row filters the page, and a mover line does the same. + * + * The width buttons matter here: the strip folds from seven columns to four to + * two, and the trends grid from three columns to two to one, at container + * widths the layout's content column does not have in this page. + */ +const WIDTHS = [ + { label: "Full", value: null }, + { label: "1400px", value: 1400 }, + { label: "1100px", value: 1100 }, + { label: "820px", value: 820 }, +] as const + +const SCENARIO_LABEL = { + healthy7d: "healthy · 7d", + regression24h: "regression · 24h", +} satisfies Record + +export function AgentOverviewLab() { + // One timestamp for the life of the mount: the whole fixture is derived from + // it, and a re-derived "now" while you look at a spacing change is noise. + const [nowMs] = useState(() => Date.now()) + const [scenario, setScenario] = useState("healthy7d") + const [search, setSearch] = useState({}) + const [width, setWidth] = useState(null) + // The page's own shape: the tab belongs to whoever issues the list read. + const [topSessionTab, setTopSessionTab] = useState("cost") + + const fixture = useMemo(() => buildOverviewFixture(scenario, nowMs), [scenario, nowMs]) + const data = useMemo( + () => buildAgentOverviewData({ ...fixture.input, compare: compareEnabled(search) }), + [fixture, search], + ) + + const onSearchChange = (patch: Partial) => + setSearch((previous) => ({ ...previous, ...patch })) + + return ( +
+
+ {OVERVIEW_SCENARIOS.map((option) => ( + + ))} + + {WIDTHS.map((option) => ( + + ))} + + {JSON.stringify(search)} + +
+ + {/* `@container/page` because the page's breakpoints are container queries + against the layout's content column, which is not mounted here. */} +
+ +
+
+ ) +} diff --git a/apps/web/src/lab/registry.ts b/apps/web/src/lab/registry.ts index c50c661d7..e705c9751 100644 --- a/apps/web/src/lab/registry.ts +++ b/apps/web/src/lab/registry.ts @@ -99,6 +99,14 @@ export const LAB_ENTRIES: ReadonlyArray = [ kind: "lab", session: "none", }, + { + path: "/lab/agent-overview", + title: "Agent overview", + description: + "The `/agent-sessions/overview` board over two synthetic weeks — a healthy one, and a day whose 14:00 step lifts the error rate, the tool calls per session and the cost per session while the cache-read band collapses.", + kind: "lab", + session: "none", + }, { path: "/lab/agent-sessions", title: "Agent sessions list", diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.test.ts b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts new file mode 100644 index 000000000..2afb60bee --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-analytics.test.ts @@ -0,0 +1,532 @@ +import { describe, expect, it } from "vitest" + +import { + EMPTY_OVERVIEW_MEASURES, + OVERVIEW_MOVER_LIMIT, + buildAgentOverviewData, + buildBreakdownRows, + buildModelMix, + buildMovers, + buildOverviewCharts, + buildOverviewSeries, + buildOverviewTiles, + cacheHitRatio, + costPerSession, + formatOverviewCount, + formatPerSession, + llmErrorRate, + overviewDelta, + overviewScopeSummary, + pricedShare, + sessionErrorRate, + shiftOverviewSeries, + tokenBandValues, + toolErrorRate, + tokensPerSession, + type OverviewMeasurePoint, + type OverviewMeasures, +} from "./overview-analytics" + +const HOUR = 3_600_000 +const MINUTE = 60_000 + +const measures = (overrides: Partial): OverviewMeasures => ({ + ...EMPTY_OVERVIEW_MEASURES, + ...overrides, +}) + +describe("derivations", () => { + it("divides by zero as zero rather than as NaN", () => { + const empty = EMPTY_OVERVIEW_MEASURES + for (const value of [ + sessionErrorRate(empty), + llmErrorRate(empty), + toolErrorRate(empty), + costPerSession(empty), + tokensPerSession(empty), + cacheHitRatio(empty), + pricedShare(empty), + ]) { + expect(value).toBe(0) + } + }) + + it("divides the LLM error rate by the raw span population, not the netted volume", () => { + // A mirrored call that failed twice reads above 100% against `llmCalls`. + const row = measures({ llmCalls: 5, llmCallSpans: 10, erroredLlmCalls: 2 }) + expect(llmErrorRate(row)).toBe(0.2) + }) + + it("measures the cache hit ratio against everything that could have been a prompt read", () => { + expect(cacheHitRatio(measures({ inputTokens: 30, cacheReadTokens: 70 }))).toBe(0.7) + }) + + it("divides the priced share by the netted volume the server priced", () => { + // 47 of 50 netted calls carried a price. The 59 spans behind them did not + // each need one, and dividing by those would read as 80% coverage. + expect(pricedShare(measures({ llmCalls: 50, llmCallSpans: 59, pricedLlmCalls: 47 }))).toBe(0.94) + }) +}) + +describe("tokenBandValues", () => { + it("splits the five bands when they carry anything", () => { + const bands = tokenBandValues( + measures({ + tokens: 100, + inputTokens: 40, + cacheReadTokens: 30, + cacheWriteTokens: 10, + outputTokens: 15, + reasoningTokens: 5, + }), + ) + expect(bands).toEqual({ + input: 40, + cacheRead: 30, + cacheWrite: 10, + output: 15, + reasoning: 5, + total: 0, + }) + }) + + it("falls back to one band for a row materialized before the bucket columns", () => { + const bands = tokenBandValues(measures({ tokens: 900 })) + expect(bands.total).toBe(900) + expect(bands.input).toBe(0) + }) + + it("leaves every band at zero when there are no tokens at all", () => { + expect(tokenBandValues(EMPTY_OVERVIEW_MEASURES).total).toBe(0) + }) +}) + +describe("overviewDelta", () => { + it("moves a rate in percentage points, never in percent", () => { + const delta = overviewDelta(0.024, 0.26, { unit: "points", riseIs: "bad" }) + expect(delta?.pp).toBeCloseTo(23.6, 5) + expect(delta?.percent).toBeNull() + expect(delta?.text).toBe("+23.6pp") + expect(delta?.tone).toBe("bad") + }) + + it("grades a fall in a rise-is-bad metric as good", () => { + expect(overviewDelta(0.26, 0.024, { unit: "points", riseIs: "bad" })?.tone).toBe("good") + }) + + it("keeps a totals metric neutral in both directions", () => { + expect(overviewDelta(100, 180, { unit: "percent", riseIs: "neutral" })?.tone).toBe("neutral") + expect(overviewDelta(180, 100, { unit: "percent", riseIs: "neutral" })?.tone).toBe("neutral") + }) + + it("grades a rise in a rise-is-good metric as good", () => { + expect(overviewDelta(0.12, 0.71, { unit: "points", riseIs: "good" })?.tone).toBe("good") + }) + + it("refuses a percentage against a window of zero", () => { + expect(overviewDelta(0, 42, { unit: "percent", riseIs: "bad" })).toBeNull() + }) + + it("reads a move too small to matter as flat and neutral", () => { + const delta = overviewDelta(0.2, 0.2001, { unit: "points", riseIs: "bad" }) + expect(delta?.direction).toBe("flat") + expect(delta?.tone).toBe("neutral") + }) + + it("moves a duration by a duration and still reports its percent", () => { + const delta = overviewDelta(96_000, 227_000, { unit: "duration", riseIs: "bad" }) + expect(delta?.absolute).toBe(131_000) + expect(delta?.text).toBe("+2m 11s") + expect(delta?.percent).toBeCloseTo(1.3646, 3) + }) + + it("reads a duration move in the Sessions list's own clock units", () => { + expect(overviewDelta(96_000, 150_580, { unit: "duration", riseIs: "bad" })?.text).toBe("+55s") + }) + + it("signs a fall", () => { + expect(overviewDelta(200, 100, { unit: "percent", riseIs: "neutral" })?.text).toBe("-50%") + }) +}) + +describe("formatters", () => { + it("prints counts in full up to a million and compacts past it", () => { + expect(formatOverviewCount(1243)).toBe((1243).toLocaleString()) + expect(formatOverviewCount(2_400_000)).toBe("2.4M") + }) + + it("keeps a decimal on a per-session ratio while it has one to keep", () => { + expect(formatPerSession(6.34)).toBe("6.3") + expect(formatPerSession(420)).toBe((420).toLocaleString()) + }) + +}) + +describe("buildOverviewTiles", () => { + const current = measures({ + sessions: 100, + erroredSessions: 12, + cost: 40, + tokens: 1_000, + toolCalls: 500, + llmCalls: 200, + llmCallSpans: 236, + pricedLlmCalls: 188, + sessionDurationP50Ms: 40_000, + sessionDurationP95Ms: 90_000, + }) + const previous = measures({ sessions: 80, erroredSessions: 4, cost: 40, tokens: 800 }) + + it("builds seven tiles in the strip's order", () => { + expect( + buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }).map((t) => t.id), + ).toEqual([ + "sessions", + "cost", + "costPerSession", + "tokens", + "errorRate", + "toolCallsPerSession", + "durationP95", + ]) + }) + + it("drops every delta when the comparison is off", () => { + const tiles = buildOverviewTiles(current, previous, { compare: false, windowLabel: "7d" }) + expect(tiles.every((tile) => tile.delta === null)).toBe(true) + }) + + it("drops every delta when the previous window ran no sessions at all", () => { + // Not even the point-valued ones: 0% to 12% against an empty window is + // the first reading there is, not a 12-point rise. + const tiles = buildOverviewTiles(current, EMPTY_OVERVIEW_MEASURES, { + compare: true, + windowLabel: "7d", + }) + expect(tiles.every((tile) => tile.delta === null)).toBe(true) + const charts = buildOverviewCharts(current, EMPTY_OVERVIEW_MEASURES, { + compare: true, + modelMix: { models: [], points: [] }, + }) + expect(charts.every((chart) => chart.delta === null)).toBe(true) + }) + + it("reads the error rate in points once there is a window to compare against", () => { + const tiles = buildOverviewTiles(current, measures({ sessions: 80, erroredSessions: 2 }), { + compare: true, + windowLabel: "7d", + }) + expect(tiles.find((tile) => tile.id === "errorRate")?.delta?.text).toBe("+9.5pp") + }) + + it("grades cost per session but leaves the cost total neutral", () => { + const tiles = buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }) + const byId = new Map(tiles.map((tile) => [tile.id, tile])) + expect(byId.get("cost")?.delta?.tone).toBe("neutral") + // $0.50 → $0.40 a session is an improvement even though the bill held. + expect(byId.get("costPerSession")?.delta?.tone).toBe("good") + }) + + it("states the priced coverage under the cost tile", () => { + const tiles = buildOverviewTiles(current, previous, { compare: true, windowLabel: "7d" }) + expect(tiles.find((tile) => tile.id === "cost")?.sub).toBe("priced 94%") + }) +}) + +describe("buildOverviewSeries", () => { + it("derives every per-bucket reading and reads an empty bucket as zero", () => { + const [busy, quiet] = buildOverviewSeries([ + { + bucket: 1_000, + ...measures({ + sessions: 10, + erroredSessions: 1, + cost: 5, + tokens: 1_000, + inputTokens: 300, + cacheReadTokens: 700, + toolCalls: 40, + erroredToolCalls: 4, + llmCalls: 80, + llmCallSpans: 100, + erroredLlmCalls: 5, + sessionDurationP50Ms: 1_000, + sessionDurationP95Ms: 4_000, + }), + }, + { bucket: 2_000, ...EMPTY_OVERVIEW_MEASURES }, + ]) + expect(busy.costPerSession).toBe(0.5) + expect(busy.tokensPerSession).toBe(100) + expect(busy.toolCallsPerSession).toBe(4) + expect(busy.llmCallsPerSession).toBe(8) + expect(busy.sessionErrorRate).toBe(0.1) + expect(busy.llmErrorRate).toBe(0.05) + expect(busy.toolErrorRate).toBe(0.1) + expect(busy.cacheHitRatio).toBe(0.7) + expect(busy.sessionP95Ms).toBe(4_000) + expect(busy.tokenBands.cacheRead).toBe(70) + expect(Object.values(quiet.tokenBands).every((value) => value === 0)).toBe(true) + expect(quiet.costPerSession).toBe(0) + }) +}) + +describe("shiftOverviewSeries", () => { + const START = Date.UTC(2026, 8, 4, 0, 0, 0) + + it("moves the previous period onto the current period's axis", () => { + const windowMs = 7 * 24 * HOUR + const points = buildOverviewSeries([{ bucket: START - windowMs, ...EMPTY_OVERVIEW_MEASURES }]) + const shifted = shiftOverviewSeries(points, { startMs: START, windowMs, bucketMs: 6 * HOUR }) + expect(shifted[0].bucket).toBe(START) + }) + + it("lands on the bucket grid for a window that is not a whole number of buckets", () => { + // The page's own default: "7d" is calendar-aligned, so it runs from a + // midnight to a now floored to the quarter hour — 6d 14h 15m of 6h + // buckets. Shifted by that raw length, every ghost point would sit 2h 15m + // off an axis matched by equality. + const bucketMs = 6 * HOUR + const windowMs = 6 * 24 * HOUR + 14 * HOUR + 15 * MINUTE + const previousStart = Math.floor((START - windowMs) / bucketMs) * bucketMs + const previous = buildOverviewSeries( + Array.from({ length: 27 }, (_, index) => ({ + bucket: previousStart + index * bucketMs, + ...EMPTY_OVERVIEW_MEASURES, + })), + ) + const shifted = shiftOverviewSeries(previous, { startMs: START, windowMs, bucketMs }) + expect(shifted.map((point) => point.bucket)).toEqual( + Array.from({ length: 27 }, (_, index) => START + index * bucketMs), + ) + }) +}) + +describe("buildModelMix", () => { + const rows = [1, 2, 3, 4, 5, 6, 7].flatMap((rank) => + [0, 1].map((bucket) => ({ + bucket, + model: `model-${rank}`, + llmCallSpans: 100 - rank * 10, + })), + ) + + it("keeps the five busiest models and folds the tail into one grey band", () => { + const mix = buildModelMix(rows) + expect(mix.models).toEqual(["model-1", "model-2", "model-3", "model-4", "model-5", "other"]) + }) + + it("stacks each bucket to one", () => { + const mix = buildModelMix(rows) + for (const point of mix.points) { + const total = mix.models.reduce((sum, model) => sum + point.shares[model], 0) + expect(total).toBeCloseTo(1, 10) + } + }) + + it("leaves out the other band when nothing was folded", () => { + expect(buildModelMix([{ bucket: 0, model: "solo", llmCallSpans: 4 }]).models).toEqual(["solo"]) + }) + + it("plots the read's own other band once, however busy it is", () => { + // The warehouse folds its tail under the same key, so `other` can be the + // busiest band on the chart and still be one band. + const withServerOther = [ + { bucket: 0, model: "other", llmCallSpans: 900 }, + ...[1, 2, 3, 4, 5, 6].map((rank) => ({ + bucket: 0, + model: `model-${rank}`, + llmCallSpans: 100 - rank * 10, + })), + ] + const mix = buildModelMix(withServerOther) + expect(mix.models.filter((model) => model === "other")).toEqual(["other"]) + const total = mix.models.reduce((sum, model) => sum + mix.points[0].shares[model], 0) + expect(total).toBeCloseTo(1, 10) + }) + + it("has no points and no models for a window that ran nothing", () => { + expect(buildModelMix([])).toEqual({ models: [], points: [] }) + }) +}) + +describe("buildBreakdownRows", () => { + const entries = [ + { + key: "opus", + current: measures({ sessions: 100, erroredSessions: 10, cost: 60, tokens: 1_000, llmCalls: 200 }), + previous: measures({ sessions: 80, erroredSessions: 4, cost: 40 }), + }, + { + key: "", + current: measures({ sessions: 50, erroredSessions: 0, cost: 40, tokens: 500 }), + previous: EMPTY_OVERVIEW_MEASURES, + }, + ] + + it("measures the cost share against the rows it is showing", () => { + const rows = buildBreakdownRows("model", entries) + expect(rows[0].shareOfCost).toBe(0.6) + expect(rows[1].shareOfCost).toBe(0.4) + }) + + it("names the unattributed key rather than hiding it", () => { + expect(buildBreakdownRows("model", entries)[1].label).toBe("Unattributed") + }) + + it("reports the session error rate for a usage dimension", () => { + const rows = buildBreakdownRows("model", entries) + expect(rows[0].errorRate).toBe(0.1) + expect(rows[0].errorRateDelta?.pp).toBeCloseTo(5, 5) + expect(rows[0].errorRateDelta?.text).toBe("+5.0pp") + expect(rows[0].errorRateDelta?.tone).toBe("bad") + }) + + it("reports the CALL error rate for the tool dimension", () => { + const rows = buildBreakdownRows("tool", [ + { + key: "run_tests", + current: measures({ sessions: 20, toolCalls: 100, erroredToolCalls: 28 }), + previous: measures({ sessions: 20, toolCalls: 50, erroredToolCalls: 2 }), + }, + ]) + expect(rows[0].errorRate).toBe(0.28) + expect(rows[0].errorRateDelta?.pp).toBeCloseTo(24, 5) + }) + + it("has no move to show for a key the previous window never saw", () => { + expect(buildBreakdownRows("model", entries)[1].errorRateDelta).toBeNull() + }) +}) + +describe("buildMovers", () => { + const quiet = { + key: "quiet", + current: measures({ sessions: 9, erroredSessions: 9 }), + previous: measures({ sessions: 9 }), + } + const failing = { + key: "opus", + current: measures({ sessions: 100, llmCallSpans: 1_000, erroredLlmCalls: 161 }), + previous: measures({ sessions: 100, llmCallSpans: 1_000, erroredLlmCalls: 19 }), + } + const pricier = { + key: "gpt", + current: measures({ sessions: 100, cost: 40 }), + previous: measures({ sessions: 100, cost: 28 }), + } + + it("drops a key too small to read in either window", () => { + expect(buildMovers([{ dimension: "model", entries: [quiet] }])).toEqual([]) + }) + + it("ranks a rate's points above a ratio's percent", () => { + const movers = buildMovers([{ dimension: "model", entries: [failing, pricier] }]) + expect(movers.map((mover) => mover.key)).toEqual(["opus", "gpt"]) + expect(movers[0].metric).toBe("llmErrorRate") + expect(movers[0].deltaText).toBe("+14.2pp") + expect(movers[0].tone).toBe("bad") + }) + + it("keeps one line per key, its worst metric", () => { + const both = { + key: "opus", + current: measures({ sessions: 100, cost: 80, llmCallSpans: 1_000, erroredLlmCalls: 161 }), + previous: measures({ sessions: 100, cost: 40, llmCallSpans: 1_000, erroredLlmCalls: 19 }), + } + const movers = buildMovers([{ dimension: "model", entries: [both] }]) + expect(movers).toHaveLength(1) + expect(movers[0].metric).toBe("llmErrorRate") + }) + + it("scores the tool error rate only under the tool dimension", () => { + const entry = { + key: "run_tests", + current: measures({ sessions: 100, toolCalls: 100, erroredToolCalls: 28 }), + previous: measures({ sessions: 100, toolCalls: 100, erroredToolCalls: 4 }), + } + expect(buildMovers([{ dimension: "tool", entries: [entry] }])[0]?.metric).toBe("toolErrorRate") + // Under `model` the tool measures are structurally zero, so nothing ranks. + expect(buildMovers([{ dimension: "model", entries: [entry] }])).toEqual([]) + }) + + it("shows at most six lines however many dimensions moved", () => { + const entries = Array.from({ length: 5 }, (_, index) => ({ + ...failing, + key: `key-${index}`, + })) + const movers = buildMovers([ + { dimension: "model", entries }, + { dimension: "agent", entries }, + ]) + expect(movers).toHaveLength(OVERVIEW_MOVER_LIMIT) + }) +}) + +describe("overviewScopeSummary", () => { + it("names the three populations the rest of the board divides by", () => { + expect(overviewScopeSummary(measures({ sessions: 1_284, llmCalls: 10_842, toolCalls: 8_101 }))).toBe( + `${(1284).toLocaleString()} sessions · ${(10842).toLocaleString()} LLM calls · ${(8101).toLocaleString()} tool calls`, + ) + }) +}) + +describe("buildAgentOverviewData", () => { + const BUCKET_MS = 6 * HOUR + // A ragged window over an epoch-aligned grid — the page's own default shape. + const START = Date.UTC(2026, 8, 4, 0, 0, 0) + const WINDOW_MS = 6 * 24 * HOUR + 14 * HOUR + 15 * MINUTE + const BUCKETS = 27 + const previousStart = Math.floor((START - WINDOW_MS) / BUCKET_MS) * BUCKET_MS + const points = (first: number, sessions: number): ReadonlyArray => + Array.from({ length: BUCKETS }, (_, index) => ({ + bucket: first + index * BUCKET_MS, + ...measures({ sessions }), + })) + + const moved = { + key: "opus", + current: measures({ sessions: 100, llmCallSpans: 1_000, erroredLlmCalls: 161 }), + previous: measures({ sessions: 100, llmCallSpans: 1_000, erroredLlmCalls: 19 }), + } + + const input = { + current: measures({ + sessions: 100, + cost: 40, + llmCalls: 100, + llmCallSpans: 118, + pricedLlmCalls: 94, + }), + previous: measures({ sessions: 80, cost: 40 }), + series: points(START, 10), + previousSeries: points(previousStart, 8), + modelMix: [{ bucket: START, model: "opus", llmCallSpans: 10 }], + breakdowns: [{ dimension: "model" as const, entries: [moved], totalKeys: 3 }], + bucketSeconds: BUCKET_MS / 1_000, + windowMs: { startMs: START, endMs: START + WINDOW_MS }, + windowLabel: "7d", + } + + it("puts every previous bucket under the current bucket it is compared with", () => { + const data = buildAgentOverviewData({ ...input, compare: true }) + expect(data.previousSeries.map((point) => point.bucket)).toEqual( + data.series.map((point) => point.bucket), + ) + }) + + it("drops the previous series entirely when the comparison is off", () => { + expect(buildAgentOverviewData({ ...input, compare: false }).previousSeries).toEqual([]) + }) + + it("ranks movers against the previous window, and nothing without one", () => { + expect(buildAgentOverviewData({ ...input, compare: true }).movers).toHaveLength(1) + expect(buildAgentOverviewData({ ...input, compare: false }).movers).toEqual([]) + }) + + it("builds nine charts and the priced coverage line", () => { + const data = buildAgentOverviewData({ ...input, compare: true }) + expect(data.charts).toHaveLength(9) + expect(data.coverage.share).toBe(0.94) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-analytics.ts b/apps/web/src/lib/agent-sessions/overview-analytics.ts new file mode 100644 index 000000000..ee637fbe2 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-analytics.ts @@ -0,0 +1,1044 @@ +// The view model behind `/agent-sessions/overview`: every number the board +// prints, derived from the two reads that produce it, with no React and no wire +// shape in sight. +// +// Three rules shape this module. +// +// The API reports COUNTS; the page reads RATIOS — cost per session, tokens per +// session, an error rate. Every one of those divides, and a window with no +// sessions in it divides by zero, so `ratio` is the only division here and it +// answers 0 rather than NaN. A tile that reads "NaN" is worse than one that +// reads "0". +// +// Durations arrive in milliseconds (the adapter converts the wire's +// nanoseconds), and quantiles never fold: the window's p95 is the summary's own +// un-bucketed figure, never the mean of the buckets'. +// +// A delta is expressed in the unit its metric is read in. A rate moves in +// percentage POINTS — 2% to 26% is "up 24 points", not "up 1200%" — a ratio +// moves in percent, and a duration moves by a duration. + +import { formatErrorRate, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { formatSessionDuration } from "@maple/ui/lib/replay-format" + +import { formatCost } from "./session-summary" +import { OVERVIEW_DIMENSIONS, type OverviewDimension } from "./overview-search" + +/* ------------------------------------------------------------------------------------------------- + * Measures — the API's numbers, in the units the client reads + * -----------------------------------------------------------------------------------------------*/ + +/** + * What every overview read reports, so a tile, a point on a chart and a table + * row are the same numbers under different groupings. + * + * A subset of the wire's `AiOverviewMeasures`: the session quantiles are + * milliseconds here rather than nanoseconds, and the per-call ones are dropped + * because nothing on the board reads them. See + * `api/warehouse/ai-agent-overview.ts`. + */ +export interface OverviewMeasures { + readonly sessions: number + readonly erroredSessions: number + /** Model calls, netted — a wrapper's roll-up, a gateway's mirror and a + * provider retry of one call are one call. */ + readonly llmCalls: number + /** Model-call SPANS, counted raw. The denominator of the LLM error rate. */ + readonly llmCallSpans: number + readonly erroredLlmCalls: number + readonly toolCalls: number + readonly erroredToolCalls: number + readonly cost: number + /** Netted model calls that carried a price — the coverage behind `cost`. */ + readonly pricedLlmCalls: number + readonly tokens: number + readonly inputTokens: number + readonly cacheReadTokens: number + readonly cacheWriteTokens: number + readonly outputTokens: number + readonly reasoningTokens: number + readonly sessionDurationP50Ms: number + readonly sessionDurationP95Ms: number +} + +export const EMPTY_OVERVIEW_MEASURES: OverviewMeasures = { + sessions: 0, + erroredSessions: 0, + llmCalls: 0, + llmCallSpans: 0, + erroredLlmCalls: 0, + toolCalls: 0, + erroredToolCalls: 0, + cost: 0, + pricedLlmCalls: 0, + tokens: 0, + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + sessionDurationP50Ms: 0, + sessionDurationP95Ms: 0, +} + +/** One bucket of a summary series. `bucket` is epoch milliseconds. */ +export interface OverviewMeasurePoint extends OverviewMeasures { + readonly bucket: number +} + +/** One key of a breakdown, over both windows. */ +export interface OverviewBreakdownEntry { + /** `''` is a real key — a span carrying no value for this dimension. */ + readonly key: string + readonly current: OverviewMeasures + readonly previous: OverviewMeasures +} + +/** One (bucket, model) pair of the model-mix read. */ +export interface OverviewModelMixRow { + readonly bucket: number + readonly model: string + readonly llmCallSpans: number +} + +/* ------------------------------------------------------------------------------------------------- + * Derivations + * -----------------------------------------------------------------------------------------------*/ + +/** The only division in this module. A window that ran nothing reads 0. */ +const ratio = (numerator: number, denominator: number): number => + denominator > 0 ? numerator / denominator : 0 + +export const sessionErrorRate = (m: OverviewMeasures): number => ratio(m.erroredSessions, m.sessions) + +/** `erroredLlmCalls / llmCallSpans` and never `/ llmCalls`: the two populations + * differ by every mirror and wrapper the netting collapses. */ +export const llmErrorRate = (m: OverviewMeasures): number => ratio(m.erroredLlmCalls, m.llmCallSpans) + +export const toolErrorRate = (m: OverviewMeasures): number => ratio(m.erroredToolCalls, m.toolCalls) + +export const costPerSession = (m: OverviewMeasures): number => ratio(m.cost, m.sessions) +export const tokensPerSession = (m: OverviewMeasures): number => ratio(m.tokens, m.sessions) +export const toolCallsPerSession = (m: OverviewMeasures): number => ratio(m.toolCalls, m.sessions) +export const llmCallsPerSession = (m: OverviewMeasures): number => ratio(m.llmCalls, m.sessions) + +/** Cache reads over everything that could have been a prompt read. */ +export const cacheHitRatio = (m: OverviewMeasures): number => + ratio(m.cacheReadTokens, m.inputTokens + m.cacheReadTokens) + +/** + * `cost` is 0 for "nobody priced it" and not for "free" — this is how much of + * the window it actually covers. + * + * Over `llmCalls` and never `llmCallSpans` — the mirror image of the LLM error + * rate above. The server nets the priced calls exactly as it nets the volume, + * so the two are one population and a fully priced window reads 100% rather + * than the netting factor. + */ +export const pricedShare = (m: OverviewMeasures): number => ratio(m.pricedLlmCalls, m.llmCalls) + +/* ------------------------------------------------------------------------------------------------- + * Token bands + * -----------------------------------------------------------------------------------------------*/ + +export const OVERVIEW_TOKEN_BANDS = ["input", "cacheRead", "cacheWrite", "output", "reasoning"] as const +export type OverviewTokenBand = (typeof OVERVIEW_TOKEN_BANDS)[number] + +/** + * The band a row falls back to. Rows materialized before the bucket columns + * existed carry a total and five zeros; showing five empty bands over a + * non-zero total would read as "no tokens". + */ +export const OVERVIEW_TOKEN_FALLBACK_BAND = "total" +export type OverviewTokenBandKey = OverviewTokenBand | typeof OVERVIEW_TOKEN_FALLBACK_BAND + +export const OVERVIEW_TOKEN_BAND_KEYS = [...OVERVIEW_TOKEN_BANDS, OVERVIEW_TOKEN_FALLBACK_BAND] as const + +const emptyBands = (): Record => ({ + input: 0, + cacheRead: 0, + cacheWrite: 0, + output: 0, + reasoning: 0, + total: 0, +}) + +/** Raw token counts per band, with the fallback applied. */ +export function tokenBandValues(m: OverviewMeasures): Record { + const bands = emptyBands() + const split = m.inputTokens + m.cacheReadTokens + m.cacheWriteTokens + m.outputTokens + m.reasoningTokens + if (split === 0) { + bands.total = m.tokens + return bands + } + bands.input = m.inputTokens + bands.cacheRead = m.cacheReadTokens + bands.cacheWrite = m.cacheWriteTokens + bands.output = m.outputTokens + bands.reasoning = m.reasoningTokens + return bands +} + +/* ------------------------------------------------------------------------------------------------- + * Deltas + * -----------------------------------------------------------------------------------------------*/ + +export type DeltaDirection = "up" | "down" | "flat" +/** How the move reads, not which way it went: a rise in cost is `bad`, a rise + * in sessions is `neutral`, a rise in cache hits is `good`. */ +export type DeltaTone = "good" | "bad" | "neutral" +/** Which unit the change is expressed in. */ +export type DeltaUnit = "percent" | "points" | "duration" + +/** The colour a graded move is drawn in. A neutral move is just a number. */ +export const deltaToneClass = (tone: DeltaTone): string => + tone === "bad" + ? "text-[var(--severity-error)]" + : tone === "good" + ? "text-[var(--severity-info)]" + : "text-muted-foreground" + +export interface OverviewDelta { + /** `after - before`, in the metric's own unit (ms for durations). */ + readonly absolute: number + /** Fractional change (0.43 = +43%); `null` against a zero baseline. */ + readonly percent: number | null + /** Percentage-point change (24 = +24pp); `null` unless the metric is a rate. */ + readonly pp: number | null + readonly direction: DeltaDirection + readonly tone: DeltaTone + /** As a tile prints it, sign included: `+43%`, `24.0pp`, `+2.4s`. */ + readonly text: string +} + +const FLAT_POINTS = 0.05 +const FLAT_PERCENT = 0.001 +/** Half the clock's own resolution: a move the duration formatter renders as + * `0s` is flat, not a signed zero. */ +const FLAT_MS = 500 + +const signed = (value: number, text: string): string => (value < 0 ? `-${text}` : `+${text}`) + +/** + * The change against the previous window. + * + * `null` where there is no reading to give: a percentage against a baseline of + * zero is "up ∞%", which is not a number anybody acts on. + */ +export function overviewDelta( + before: number, + after: number, + options: { unit: DeltaUnit; riseIs: DeltaTone }, +): OverviewDelta | null { + if (!Number.isFinite(before) || !Number.isFinite(after)) return null + const absolute = after - before + const flatAt = options.unit === "points" ? FLAT_POINTS / 100 : options.unit === "duration" ? FLAT_MS : 0 + const percent = before === 0 ? null : absolute / before + + if (options.unit === "percent" && percent === null) return null + + const direction: DeltaDirection = + options.unit === "percent" + ? Math.abs(percent ?? 0) < FLAT_PERCENT + ? "flat" + : absolute > 0 + ? "up" + : "down" + : Math.abs(absolute) < flatAt + ? "flat" + : absolute > 0 + ? "up" + : "down" + + const tone: DeltaTone = + direction === "flat" || options.riseIs === "neutral" + ? "neutral" + : direction === "up" + ? options.riseIs + : options.riseIs === "bad" + ? "good" + : "bad" + + if (options.unit === "points") { + const pp = absolute * 100 + return { + absolute, + percent: null, + pp, + direction, + // A move too small to read is not a signed zero: "+0.0pp" reads as a + // rise that rounded away, which is a different claim from "flat". + text: direction === "flat" ? "0pp" : signed(pp, `${Math.abs(pp).toFixed(1)}pp`), + tone, + } + } + if (options.unit === "duration") { + return { + absolute, + percent, + pp: null, + direction, + text: direction === "flat" ? "0s" : signed(absolute, formatSessionDuration(Math.abs(absolute))), + tone, + } + } + return { + absolute, + percent, + pp: null, + direction, + text: direction === "flat" ? "0%" : signed(absolute, formatPercent(Math.abs(percent ?? 0))), + tone, + } +} + +/** + * The delta helper a strip or a grid reads its moves with. + * + * `null` for every reading while the comparison is off, and equally while the + * previous window ran NO sessions: a rate that went from 0% to 26% against an + * empty window did not rise 26 points, it is the first measurement there is. + * That holds for the point-valued readings too, which is the case a raw + * subtraction gets wrong — the ratios already answer `null` on their own. + */ +function deltaReader( + previous: OverviewMeasures, + compare: boolean, +): (before: number, after: number, unit: DeltaUnit, riseIs: DeltaTone) => OverviewDelta | null { + const hasBaseline = compare && previous.sessions > 0 + return (before, after, unit, riseIs) => + hasBaseline ? overviewDelta(before, after, { unit, riseIs }) : null +} + +/* ------------------------------------------------------------------------------------------------- + * Formatters + * -----------------------------------------------------------------------------------------------*/ + +/** + * Counts read as themselves up to a million. "1.2K" and "1,243" are the same + * number to a reader; compaction starts where the exact digits stop being + * something anyone holds in their head. + */ +export function formatOverviewCount(value: number): string { + return Math.abs(value) >= 1_000_000 ? formatNumber(value) : Math.round(value).toLocaleString() +} + +/** A per-session ratio: one decimal while the number is small enough to have one. */ +export function formatPerSession(value: number): string { + if (!Number.isFinite(value)) return "—" + return value >= 100 ? formatOverviewCount(value) : value.toFixed(1) +} + +/** `''` is a real breakdown key, shown as unattributed rather than hidden. */ +export const UNATTRIBUTED_LABEL = "Unattributed" +export const breakdownKeyLabel = (key: string): string => (key === "" ? UNATTRIBUTED_LABEL : key) + +/** + * What the current scope matched, in one line. + * + * The three populations the rest of the board divides by, so a reader can see + * at a glance whether a rate is measured over a thousand sessions or over four. + */ +export function overviewScopeSummary(current: OverviewMeasures): string { + return [ + `${formatOverviewCount(current.sessions)} sessions`, + `${formatOverviewCount(current.llmCalls)} LLM calls`, + `${formatOverviewCount(current.toolCalls)} tool calls`, + ].join(" · ") +} + +/* ------------------------------------------------------------------------------------------------- + * KPI tiles + * -----------------------------------------------------------------------------------------------*/ + +export const OVERVIEW_TILES = [ + "sessions", + "cost", + "costPerSession", + "tokens", + "errorRate", + "toolCallsPerSession", + "durationP95", +] as const +export type OverviewTileId = (typeof OVERVIEW_TILES)[number] + +export interface OverviewTile { + readonly id: OverviewTileId + /** The eyebrow, in the page's own words. */ + readonly label: string + readonly value: string + /** A short suffix beside the value, where one helps read it. */ + readonly unit?: string + /** `null` when the comparison is off, or when there is no reading to give. */ + readonly delta: OverviewDelta | null + /** The second half of the delta line: what the number is made of. */ + readonly sub: string +} + +/** + * The seven tiles, left to right. + * + * Totals are neutral — more sessions is neither good nor bad news, and a bill + * that rose because usage rose is not a regression. What is graded is the unit + * economics and the failures: cost per session, tokens per session, the error + * rate, tool calls per session and the p95. + */ +export function buildOverviewTiles( + current: OverviewMeasures, + previous: OverviewMeasures, + options: { compare: boolean; windowLabel: string }, +): ReadonlyArray { + const delta = deltaReader(previous, options.compare) + + return [ + { + id: "sessions", + label: "Sessions", + value: formatOverviewCount(current.sessions), + delta: delta(previous.sessions, current.sessions, "percent", "neutral"), + sub: options.compare + ? `vs ${formatOverviewCount(previous.sessions)} prev` + : `over ${options.windowLabel}`, + }, + { + id: "cost", + label: "Cost", + value: formatCost(current.cost), + delta: delta(previous.cost, current.cost, "percent", "neutral"), + sub: `priced ${formatPercent(pricedShare(current))}`, + }, + { + id: "costPerSession", + label: "Cost / session", + value: formatCost(costPerSession(current)), + delta: delta(costPerSession(previous), costPerSession(current), "percent", "bad"), + sub: options.compare + ? `vs ${formatCost(costPerSession(previous))}` + : `${formatOverviewCount(current.sessions)} sessions`, + }, + { + id: "tokens", + label: "Tokens", + value: formatNumber(current.tokens), + unit: "tok", + delta: delta(previous.tokens, current.tokens, "percent", "neutral"), + sub: `${formatNumber(tokensPerSession(current))} / session`, + }, + { + id: "errorRate", + label: "Error rate", + value: formatErrorRate(sessionErrorRate(current)), + delta: delta(sessionErrorRate(previous), sessionErrorRate(current), "points", "bad"), + sub: `${formatOverviewCount(current.erroredSessions)} errored`, + }, + { + id: "toolCallsPerSession", + label: "Tool calls / sess", + value: formatPerSession(toolCallsPerSession(current)), + delta: delta(toolCallsPerSession(previous), toolCallsPerSession(current), "percent", "bad"), + sub: `${formatOverviewCount(current.toolCalls)} calls`, + }, + { + id: "durationP95", + label: "Duration p95", + value: formatSessionDuration(current.sessionDurationP95Ms), + delta: delta(previous.sessionDurationP95Ms, current.sessionDurationP95Ms, "duration", "bad"), + sub: `p50 ${formatSessionDuration(current.sessionDurationP50Ms)}`, + }, + ] +} + +/* ------------------------------------------------------------------------------------------------- + * The series behind the small multiples + * -----------------------------------------------------------------------------------------------*/ + +export interface OverviewSeriesPoint { + /** Epoch milliseconds, the bucket's start. */ + readonly bucket: number + readonly sessions: number + readonly costPerSession: number + readonly tokensPerSession: number + /** Tokens per session, split by band — the stacked chart's values. */ + readonly tokenBands: Record + readonly toolCallsPerSession: number + readonly sessionErrorRate: number + readonly llmErrorRate: number + readonly toolErrorRate: number + readonly sessionP50Ms: number + readonly sessionP95Ms: number + readonly llmCallsPerSession: number + readonly cacheHitRatio: number +} + +export function buildOverviewSeries( + points: ReadonlyArray, +): ReadonlyArray { + return points.map((point) => { + const bands = tokenBandValues(point) + const perSession = emptyBands() + for (const key of OVERVIEW_TOKEN_BAND_KEYS) { + perSession[key] = ratio(bands[key], point.sessions) + } + return { + bucket: point.bucket, + sessions: point.sessions, + costPerSession: costPerSession(point), + tokensPerSession: tokensPerSession(point), + tokenBands: perSession, + toolCallsPerSession: toolCallsPerSession(point), + sessionErrorRate: sessionErrorRate(point), + llmErrorRate: llmErrorRate(point), + toolErrorRate: toolErrorRate(point), + sessionP50Ms: point.sessionDurationP50Ms, + sessionP95Ms: point.sessionDurationP95Ms, + llmCallsPerSession: llmCallsPerSession(point), + cacheHitRatio: cacheHitRatio(point), + } + }) +} + +/** The bucket a moment falls in — `toStartOfInterval`, as the warehouse cuts it. */ +const bucketStart = (ms: number, bucketMs: number): number => Math.floor(ms / bucketMs) * bucketMs + +/** + * The previous period's points moved onto the current period's x-axis. + * + * The shift is a WHOLE number of buckets and never the window's own length: both + * windows are cut into epoch-aligned bucket starts, so only a multiple of the + * bucket width lands one grid on the other, and the ghost is matched to the + * subject bucket for bucket, by equality. The multiple is the one that puts the + * previous window's first bucket under the current window's first, which for a + * window measuring a whole number of buckets is simply its length — and the + * default 7d window does not: it runs from a midnight to a "now" floored to the + * quarter hour, so shifting by its own length left every ghost point minutes off + * the axis and nothing matched at all. + */ +export function shiftOverviewSeries( + points: ReadonlyArray, + window: { startMs: number; windowMs: number; bucketMs: number }, +): ReadonlyArray { + const offsetMs = + bucketStart(window.startMs, window.bucketMs) - + bucketStart(window.startMs - window.windowMs, window.bucketMs) + return points.map((point) => ({ ...point, bucket: point.bucket + offsetMs })) +} + +/* ------------------------------------------------------------------------------------------------- + * Model mix + * -----------------------------------------------------------------------------------------------*/ + +/** Models plotted as their own band; everything past this is folded. */ +export const OVERVIEW_MODEL_MIX_LIMIT = 5 +export const OVERVIEW_MODEL_MIX_OTHER = "other" + +export interface OverviewModelMixPoint { + readonly bucket: number + /** Spans per band key, over the models {@link OverviewModelMix.models} names. */ + readonly spans: Record + /** Each band's share of the bucket, 0–1 — the 100% stack. */ + readonly shares: Record +} + +export interface OverviewModelMix { + /** The plotted bands, busiest first, with `other` last when the tail exists. */ + readonly models: ReadonlyArray + readonly points: ReadonlyArray +} + +/** + * The top models by span count, with the rest folded into one grey band. + * + * A line per model is unreadable past a handful and the tail is a residue + * rather than a thing; what the chart is for is noticing that one band took + * over. + */ +export function buildModelMix(rows: ReadonlyArray): OverviewModelMix { + const totals = new Map() + for (const row of rows) totals.set(row.model, (totals.get(row.model) ?? 0) + row.llmCallSpans) + + const ranked = [...totals.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([model]) => model) + const top = ranked.slice(0, OVERVIEW_MODEL_MIX_LIMIT) + const kept = new Set(top) + // The read folds its own tail, so `other` arrives as a band of its own and + // can rank inside the top: appending it again would plot it twice and + // double it in the bucket's total. + const models = + ranked.length > top.length && !kept.has(OVERVIEW_MODEL_MIX_OTHER) + ? [...top, OVERVIEW_MODEL_MIX_OTHER] + : top + + const byBucket = new Map>() + for (const row of rows) { + const band = kept.has(row.model) ? row.model : OVERVIEW_MODEL_MIX_OTHER + let bucket = byBucket.get(row.bucket) + if (bucket === undefined) { + bucket = Object.fromEntries(models.map((model) => [model, 0])) + byBucket.set(row.bucket, bucket) + } + bucket[band] += row.llmCallSpans + } + + const points = [...byBucket.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([bucket, spans]) => { + const total = models.reduce((sum, model) => sum + spans[model], 0) + const shares: Record = {} + for (const model of models) shares[model] = ratio(spans[model], total) + return { bucket, spans, shares } + }) + + return { models, points } +} + +/* ------------------------------------------------------------------------------------------------- + * Breakdown tables + * -----------------------------------------------------------------------------------------------*/ + +export interface OverviewBreakdownRow { + readonly key: string + readonly label: string + /** Share of the table's cost, 0–1 — the usage dimensions' bar. */ + readonly shareOfCost: number + /** Share of the table's tool calls, 0–1 — the tool dimension's bar. */ + readonly shareOfCalls: number + readonly sessions: number + readonly llmCalls: number + readonly tokensPerSession: number + readonly cost: number + readonly costPerSession: number + readonly toolCalls: number + readonly toolErrors: number + /** Tool calls that failed for the `tool` dimension, sessions that failed for + * every other — the failure the dimension can actually attribute. */ + readonly errorRate: number + /** The same rate's move, in percentage points; `null` where the key did not + * appear in the previous window. */ + readonly errorRateDelta: OverviewDelta | null +} + +export interface OverviewBreakdown { + readonly dimension: OverviewDimension + readonly rows: ReadonlyArray + /** Distinct keys the window had, so the table can say what it is not showing. */ + readonly totalKeys: number +} + +/** A tool row's failures are its calls'; every other dimension's are its sessions'. */ +const dimensionErrorRate = (dimension: OverviewDimension, m: OverviewMeasures): number => + dimension === "tool" ? toolErrorRate(m) : sessionErrorRate(m) + +const dimensionPopulation = (dimension: OverviewDimension, m: OverviewMeasures): number => + dimension === "tool" ? m.toolCalls : m.sessions + +/** + * One table's rows, in the order the server ranked them (busiest first). + * + * Shares are of the ROWS, not of the window: the table shows at most a dozen + * keys and a bar measured against a total it is not showing would never fill. + */ +export function buildBreakdownRows( + dimension: OverviewDimension, + entries: ReadonlyArray, +): ReadonlyArray { + const totalCost = entries.reduce((sum, entry) => sum + entry.current.cost, 0) + const totalCalls = entries.reduce((sum, entry) => sum + entry.current.toolCalls, 0) + return entries.map((entry) => { + const rate = dimensionErrorRate(dimension, entry.current) + const hadPrevious = dimensionPopulation(dimension, entry.previous) > 0 + return { + key: entry.key, + label: breakdownKeyLabel(entry.key), + shareOfCost: ratio(entry.current.cost, totalCost), + shareOfCalls: ratio(entry.current.toolCalls, totalCalls), + sessions: entry.current.sessions, + llmCalls: entry.current.llmCalls, + tokensPerSession: tokensPerSession(entry.current), + cost: entry.current.cost, + costPerSession: costPerSession(entry.current), + toolCalls: entry.current.toolCalls, + toolErrors: entry.current.erroredToolCalls, + errorRate: rate, + errorRateDelta: hadPrevious + ? overviewDelta(dimensionErrorRate(dimension, entry.previous), rate, { + unit: "points", + riseIs: "bad", + }) + : null, + } + }) +} + +/* ------------------------------------------------------------------------------------------------- + * "What changed" — the movers rail + * -----------------------------------------------------------------------------------------------*/ + +/** A key this quiet moved by accident, not by regression. */ +export const OVERVIEW_MOVER_MIN_SESSIONS = 10 +export const OVERVIEW_MOVER_LIMIT = 6 + +export type OverviewMoverMetric = + | "sessionErrorRate" + | "llmErrorRate" + | "toolErrorRate" + | "costPerSession" + | "toolCallsPerSession" + | "tokensPerSession" + | "durationP95" + +export interface OverviewMover { + readonly dimension: OverviewDimension + readonly key: string + readonly label: string + readonly metric: OverviewMoverMetric + readonly metricLabel: string + /** Formatted in the metric's own unit. */ + readonly before: string + readonly after: string + readonly deltaText: string + readonly tone: DeltaTone + /** Percentage points for a rate, tenths of a percent for a ratio — the one + * scale the rail ranks on, and the magnitude bar's length. */ + readonly score: number +} + +interface MoverMetric { + readonly id: OverviewMoverMetric + readonly label: string + readonly unit: DeltaUnit + readonly riseIs: DeltaTone + readonly value: (m: OverviewMeasures) => number + readonly format: (value: number) => string + /** Absent means every dimension. */ + readonly only?: OverviewDimension +} + +const MOVER_METRICS: ReadonlyArray = [ + { + id: "sessionErrorRate", + label: "session error rate", + unit: "points", + riseIs: "bad", + value: sessionErrorRate, + format: formatErrorRate, + }, + { + id: "llmErrorRate", + label: "LLM error rate", + unit: "points", + riseIs: "bad", + value: llmErrorRate, + format: formatErrorRate, + }, + { + id: "toolErrorRate", + label: "tool error rate", + unit: "points", + riseIs: "bad", + value: toolErrorRate, + format: formatErrorRate, + only: "tool", + }, + { + id: "costPerSession", + label: "cost / session", + unit: "percent", + riseIs: "bad", + value: costPerSession, + format: formatCost, + }, + { + id: "toolCallsPerSession", + label: "tool calls / session", + unit: "percent", + riseIs: "bad", + value: toolCallsPerSession, + format: formatPerSession, + }, + { + id: "tokensPerSession", + label: "tokens / session", + unit: "percent", + riseIs: "bad", + value: tokensPerSession, + format: formatNumber, + }, + { + id: "durationP95", + label: "p95 duration", + unit: "percent", + riseIs: "bad", + value: (m) => m.sessionDurationP95Ms, + format: formatSessionDuration, + }, +] + +/** Points for a rate, a tenth of a percent for a ratio — so an 8-point jump in + * failures outranks an 80% rise in tokens, which is the order a reader wants. */ +const moverScore = (delta: OverviewDelta): number => + delta.pp !== null ? Math.abs(delta.pp) : Math.abs((delta.percent ?? 0) * 100) / 10 + +/** + * The biggest movers across every dimension, worst first. + * + * One line per key, not per metric: a key whose cost and tokens both doubled + * moved once, and printing it twice would push a different regression off the + * rail. Keys too small to read are dropped in both windows — a group that ran + * three sessions last week and four this week can post any rate at all. + */ +export function buildMovers( + breakdowns: ReadonlyArray<{ + readonly dimension: OverviewDimension + readonly entries: ReadonlyArray + }>, +): ReadonlyArray { + const best: OverviewMover[] = [] + for (const breakdown of breakdowns) { + for (const entry of breakdown.entries) { + if ( + entry.current.sessions < OVERVIEW_MOVER_MIN_SESSIONS || + entry.previous.sessions < OVERVIEW_MOVER_MIN_SESSIONS + ) { + continue + } + let winner: OverviewMover | undefined + for (const metric of MOVER_METRICS) { + if (metric.only !== undefined && metric.only !== breakdown.dimension) continue + const before = metric.value(entry.previous) + const after = metric.value(entry.current) + const delta = overviewDelta(before, after, { + unit: metric.unit, + riseIs: metric.riseIs, + }) + if (delta === null || delta.direction === "flat") continue + const score = moverScore(delta) + if (winner !== undefined && score <= winner.score) continue + winner = { + dimension: breakdown.dimension, + key: entry.key, + label: breakdownKeyLabel(entry.key), + metric: metric.id, + metricLabel: metric.label, + before: metric.format(before), + after: metric.format(after), + deltaText: delta.text, + tone: delta.tone, + score, + } + } + if (winner !== undefined) best.push(winner) + } + } + return best + .sort( + (a, b) => + b.score - a.score || + OVERVIEW_DIMENSIONS.indexOf(a.dimension) - OVERVIEW_DIMENSIONS.indexOf(b.dimension) || + a.key.localeCompare(b.key), + ) + .slice(0, OVERVIEW_MOVER_LIMIT) +} + +/* ------------------------------------------------------------------------------------------------- + * Coverage + * -----------------------------------------------------------------------------------------------*/ + +/** The one coverage line the API can answer: how much of the window has a price. */ +export interface OverviewCoverage { + readonly label: string + readonly share: number +} + +export const overviewCoverage = (current: OverviewMeasures): OverviewCoverage => ({ + label: "LLM calls with a cost", + share: pricedShare(current), +}) + +/* ------------------------------------------------------------------------------------------------- + * The nine small multiples + * -----------------------------------------------------------------------------------------------*/ + +export const OVERVIEW_CHARTS = [ + "sessions", + "costPerSession", + "tokensPerSession", + "toolCallsPerSession", + "errorRate", + "sessionDuration", + "llmCallsPerSession", + "modelMix", + "cacheHitRatio", +] as const +export type OverviewChartId = (typeof OVERVIEW_CHARTS)[number] + +export interface OverviewChartSummary { + readonly id: OverviewChartId + readonly title: string + /** The unit sub-label under the title. */ + readonly unit: string + /** The window's headline, from the un-bucketed summary — never folded from + * the buckets, because quantiles do not merge. */ + readonly value: string + readonly delta: OverviewDelta | null +} + +/** The chart headlines, in the grid's reading order. */ +export function buildOverviewCharts( + current: OverviewMeasures, + previous: OverviewMeasures, + options: { compare: boolean; modelMix: OverviewModelMix }, +): ReadonlyArray { + const delta = deltaReader(previous, options.compare) + + const leadModel = options.modelMix.models[0] + const spansOf = (model: string) => + options.modelMix.points.reduce((sum, point) => sum + point.spans[model], 0) + const mixSpans = options.modelMix.models.reduce((sum, model) => sum + spansOf(model), 0) + const leadShare = leadModel === undefined ? 0 : ratio(spansOf(leadModel), mixSpans) + + return [ + { + id: "sessions", + title: "Sessions started", + unit: "sessions / bucket", + value: formatOverviewCount(current.sessions), + delta: delta(previous.sessions, current.sessions, "percent", "neutral"), + }, + { + id: "costPerSession", + title: "Cost per session", + unit: "USD / session", + value: formatCost(costPerSession(current)), + delta: delta(costPerSession(previous), costPerSession(current), "percent", "bad"), + }, + { + id: "tokensPerSession", + title: "Tokens per session", + unit: "tokens / session, by band", + value: formatNumber(tokensPerSession(current)), + delta: delta(tokensPerSession(previous), tokensPerSession(current), "percent", "bad"), + }, + { + id: "toolCallsPerSession", + title: "Tool calls per session", + unit: "calls / session", + value: formatPerSession(toolCallsPerSession(current)), + delta: delta(toolCallsPerSession(previous), toolCallsPerSession(current), "percent", "bad"), + }, + { + id: "errorRate", + title: "Error rate", + unit: "sessions · LLM calls · tool calls", + value: formatErrorRate(sessionErrorRate(current)), + delta: delta(sessionErrorRate(previous), sessionErrorRate(current), "points", "bad"), + }, + { + id: "sessionDuration", + title: "Session duration", + unit: "p50 with p50–p95 band", + value: formatSessionDuration(current.sessionDurationP95Ms), + delta: delta(previous.sessionDurationP95Ms, current.sessionDurationP95Ms, "duration", "bad"), + }, + { + id: "llmCallsPerSession", + title: "LLM calls per session", + unit: "calls / session", + value: formatPerSession(llmCallsPerSession(current)), + delta: delta(llmCallsPerSession(previous), llmCallsPerSession(current), "percent", "neutral"), + }, + { + id: "modelMix", + title: "Model mix", + unit: "share of LLM call spans", + value: leadModel === undefined ? "—" : `${leadModel} ${formatPercent(leadShare)}`, + delta: null, + }, + { + id: "cacheHitRatio", + title: "Cache hit ratio", + unit: "cache reads / prompt tokens", + value: formatPercent(cacheHitRatio(current)), + delta: delta(cacheHitRatio(previous), cacheHitRatio(current), "points", "good"), + }, + ] +} + +/* ------------------------------------------------------------------------------------------------- + * The whole board + * -----------------------------------------------------------------------------------------------*/ + +export interface AgentOverviewData { + readonly current: OverviewMeasures + readonly previous: OverviewMeasures + readonly compare: boolean + readonly tiles: ReadonlyArray + readonly charts: ReadonlyArray + readonly series: ReadonlyArray + /** Already shifted onto the current window's axis; empty when compare is off. */ + readonly previousSeries: ReadonlyArray + readonly modelMix: OverviewModelMix + readonly movers: ReadonlyArray + readonly coverage: OverviewCoverage + /** All six, in `OVERVIEW_DIMENSIONS` order — the tabs are component state, + * and the movers rail reads every one of them anyway. */ + readonly breakdowns: ReadonlyArray + readonly bucketSeconds: number +} + +export interface AgentOverviewInput { + readonly current: OverviewMeasures + readonly previous: OverviewMeasures + readonly series: ReadonlyArray + readonly previousSeries: ReadonlyArray + readonly modelMix: ReadonlyArray + readonly breakdowns: ReadonlyArray<{ + readonly dimension: OverviewDimension + readonly entries: ReadonlyArray + readonly totalKeys: number + }> + readonly bucketSeconds: number + /** The resolved window, in epoch ms — its length is the ghost's shift. */ + readonly windowMs: { readonly startMs: number; readonly endMs: number } + /** Names the comparison in the tiles, e.g. `7d`. */ + readonly windowLabel: string + readonly compare: boolean +} + +/** Everything the view renders, from everything the reads returned. */ +export function buildAgentOverviewData(input: AgentOverviewInput): AgentOverviewData { + const modelMix = buildModelMix(input.modelMix) + const windowMs = input.windowMs.endMs - input.windowMs.startMs + return { + current: input.current, + previous: input.previous, + compare: input.compare, + tiles: buildOverviewTiles(input.current, input.previous, { + compare: input.compare, + windowLabel: input.windowLabel, + }), + charts: buildOverviewCharts(input.current, input.previous, { + compare: input.compare, + modelMix, + }), + series: buildOverviewSeries(input.series), + previousSeries: input.compare + ? shiftOverviewSeries(buildOverviewSeries(input.previousSeries), { + startMs: input.windowMs.startMs, + windowMs, + bucketMs: input.bucketSeconds * 1_000, + }) + : [], + modelMix, + // Every line on the rail is a move against the previous window; with the + // comparison off there is nothing to rank, and the rail says so. + movers: input.compare ? buildMovers(input.breakdowns) : [], + coverage: overviewCoverage(input.current), + breakdowns: input.breakdowns.map((breakdown) => ({ + dimension: breakdown.dimension, + rows: buildBreakdownRows(breakdown.dimension, breakdown.entries), + totalKeys: breakdown.totalKeys, + })), + bucketSeconds: input.bucketSeconds, + } +} diff --git a/apps/web/src/lib/agent-sessions/overview-buckets.test.ts b/apps/web/src/lib/agent-sessions/overview-buckets.test.ts new file mode 100644 index 000000000..eecf460cf --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-buckets.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest" + +import { bucketWidthLabel, smallMultipleBucketSeconds } from "./overview-buckets" + +describe("smallMultipleBucketSeconds", () => { + /** A window of `hours`, spelled the way the warehouse spells one. */ + const window = (hours: number) => { + const endMs = Date.UTC(2026, 8, 11, 0, 0, 0) + const iso = (ms: number) => new Date(ms).toISOString().replace("T", " ").slice(0, 19) + return [iso(endMs - hours * 3_600_000), iso(endMs)] as const + } + const widthOf = (hours: number) => smallMultipleBucketSeconds(...window(hours)) + + it("cuts the usual windows at a width a reader recognises", () => { + expect(widthOf(24)).toBe(3_600) + expect(widthOf(24 * 7)).toBe(21_600) + expect(widthOf(24 * 30)).toBe(86_400) + }) + + it("keeps every window inside the count a ~100px plot can separate", () => { + for (const hours of [1, 6, 12, 24, 72, 24 * 7, 24 * 30]) { + const points = (hours * 3_600) / widthOf(hours) + expect(points).toBeLessThanOrEqual(36) + } + }) + + it("stays on the five-minute grid the API's bucket bound needs", () => { + for (const hours of [0.25, 1, 6, 12, 24, 72, 24 * 7, 24 * 30, 24 * 90]) { + const width = widthOf(hours) + expect(width % 300).toBe(0) + expect(Number.isInteger(width)).toBe(true) + } + }) + + it("floors at five minutes and tops out at a day", () => { + expect(widthOf(0.25)).toBe(300) + expect(widthOf(24 * 365)).toBe(86_400) + }) +}) + +describe("bucketWidthLabel", () => { + it("names every width the grid's buckets are actually cut at", () => { + // The ladder `smallMultipleBucketSeconds` snaps to, one unit each. + const ladder = [300, 900, 1_800, 3_600, 10_800, 21_600, 43_200, 86_400] + expect(ladder.map(bucketWidthLabel)).toEqual(["5m", "15m", "30m", "1h", "3h", "6h", "12h", "1d"]) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-buckets.ts b/apps/web/src/lib/agent-sessions/overview-buckets.ts new file mode 100644 index 000000000..2cdda9044 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-buckets.ts @@ -0,0 +1,51 @@ +// The bucket width the overview's grid of small multiples is cut at, and the +// name the page prints for it. +// +// Here rather than beside the infra charts' helpers: the ladder is sized for a +// ~100px-tall plot in a nine-cell grid, which is this page's layout and nothing +// else's. + +import { toEpochMs } from "@maple/ui/lib/time-format" + +/** + * Bucket widths a small multiple can be read at: whole, recognisable steps from + * five minutes to a day, every one of them a multiple of 300 so the width also + * satisfies the API's `BucketSeconds`. + */ +const SMALL_MULTIPLE_CEILING = 86_400 +const SMALL_MULTIPLE_LADDER: ReadonlyArray = [ + 300, + 900, + 1_800, + 3_600, + 10_800, + 21_600, + 43_200, + SMALL_MULTIPLE_CEILING, +] + +/** Points a ~100px-tall plot can separate. A hundred of them land under a pixel each. */ +const SMALL_MULTIPLE_TARGET_POINTS = 30 + +/** + * Bucket width for a grid of small multiples: about thirty points, snapped up + * to the ladder above. + * + * Two things differ from `chartBucketSeconds`, which a full-width chart wants. + * The count: a plot barely a hundred pixels tall cannot show a hundred buckets. + * And the snapping: dividing a window into a hundred equal parts produces + * widths like 105 minutes, which an axis note has to call "2h" while the + * buckets are something else. A day reads at 1h, a week at 6h, a month at 1d. + */ +export function smallMultipleBucketSeconds(startTime: string, endTime: string): number { + const windowSeconds = Math.max((toEpochMs(endTime) - toEpochMs(startTime)) / 1000, 300) + const target = windowSeconds / SMALL_MULTIPLE_TARGET_POINTS + return SMALL_MULTIPLE_LADDER.find((width) => width >= target) ?? SMALL_MULTIPLE_CEILING +} + +/** The bucket width, as the Trends note states it: `15m`, `6h`, `1d`. */ +export function bucketWidthLabel(seconds: number): string { + if (seconds >= 86_400) return `${Math.round(seconds / 86_400)}d` + if (seconds >= 3_600) return `${Math.round(seconds / 3_600)}h` + return `${Math.round(seconds / 60)}m` +} diff --git a/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts b/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts new file mode 100644 index 000000000..ebbd1bbfb --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-chart-specs.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from "vitest" + +import { + EMPTY_OVERVIEW_MEASURES, + buildModelMix, + buildOverviewSeries, + type OverviewMeasurePoint, + type OverviewMeasures, + type OverviewModelMix, +} from "./overview-analytics" +import { + OVERVIEW_TICK_PADDING, + buildOverviewPlotSpec, + overviewAxisGutter, + overviewAxisTick, + type OverviewPlotInput, + type OverviewPlotSpec, +} from "./overview-chart-specs" + +const HOUR = 3_600_000 +const START = Date.UTC(2026, 8, 10, 0, 0, 0) + +const point = (index: number, overrides: Partial): OverviewMeasurePoint => ({ + ...EMPTY_OVERVIEW_MEASURES, + ...overrides, + bucket: START + index * HOUR, +}) + +const EMPTY_MIX: OverviewModelMix = { models: [], points: [] } + +const input = (overrides: Partial): OverviewPlotInput => ({ + series: [], + previousSeries: [], + modelMix: EMPTY_MIX, + ...overrides, +}) + +const series = buildOverviewSeries([ + point(0, { sessions: 10, cost: 5, tokens: 300, inputTokens: 200, cacheReadTokens: 100 }), + point(1, { sessions: 20, cost: 30, tokens: 900, inputTokens: 600, outputTokens: 300 }), +]) + +describe("buildOverviewPlotSpec", () => { + it("draws the previous period only when there is one to draw", () => { + const without = buildOverviewPlotSpec("sessions", input({ series })) + expect(without.marks.map((mark) => mark.kind)).toEqual(["line"]) + + // Already shifted onto this axis, which is what the ghost is matched by. + const withGhost = buildOverviewPlotSpec("sessions", input({ series, previousSeries: series })) + expect(withGhost.marks.map((mark) => mark.kind)).toEqual(["line", "ghost"]) + expect(withGhost.legend.at(-1)?.label).toBe("prev") + }) + + it("names no ghost when the previous points landed off this axis", () => { + // A previous series a bucket out has nothing to draw at any x the plot + // has, and a legend entry for it would promise a line that is not there. + const offAxis = buildOverviewSeries([point(-1, { sessions: 8 }), point(-2, { sessions: 6 })]) + const spec = buildOverviewPlotSpec("sessions", input({ series, previousSeries: offAxis })) + expect(spec.marks.map((mark) => mark.kind)).toEqual(["line"]) + }) + + it("tops the axis on the rung ABOVE the data, and at 1 for an empty window", () => { + // 20 sessions is itself a rung on the ladder, so the axis takes the next + // one: a flat line drawn along the top edge reads as clipped, not steady. + expect(buildOverviewPlotSpec("sessions", input({ series })).yMax).toBe(25) + expect(buildOverviewPlotSpec("toolCallsPerSession", input({ series })).yMax).toBe(1) + expect(buildOverviewPlotSpec("sessions", input({})).yMax).toBe(1) + }) + + it("pins the share charts to a full axis so a mix is read against 100%", () => { + expect(buildOverviewPlotSpec("cacheHitRatio", input({ series })).yMax).toBe(1) + expect(buildOverviewPlotSpec("modelMix", input({})).yMax).toBe(1) + }) + + it("stacks the token bands, each layer sitting on the one below", () => { + const spec = buildOverviewPlotSpec("tokensPerSession", input({ series })) + expect(spec.marks.map((mark) => mark.key)).toEqual([ + "input", + "cacheRead", + "cacheWrite", + "output", + "reasoning", + ]) + // 200 input + 100 cache read over 10 sessions: 20 then 30. + expect(spec.rows[0]?.input).toBe(20) + expect(spec.rows[0]?.cacheRead_base).toBe(20) + expect(spec.rows[0]?.cacheRead).toBe(30) + expect(spec.yMax).toBe(50) + }) + + it("falls back to one band when no bucket reported a breakdown", () => { + const flat = buildOverviewSeries([point(0, { sessions: 4, tokens: 400 })]) + const spec = buildOverviewPlotSpec("tokensPerSession", input({ series: flat })) + expect(spec.marks.map((mark) => mark.key)).toEqual(["total"]) + expect(spec.rows[0]?.total).toBe(100) + }) + + it("names the leading models in the legend and counts the rest", () => { + const mix = buildModelMix( + ["a", "b", "c", "d"].map((model, index) => ({ + bucket: START, + model, + llmCallSpans: 10 - index, + })), + ) + const spec = buildOverviewPlotSpec("modelMix", input({ modelMix: mix })) + expect(spec.marks).toHaveLength(4) + expect(spec.legend).toHaveLength(3) + expect(spec.legendMore).toBe(1) + expect(spec.legend[0]?.label).toBe("a 29%") + }) + + it("spreads the duration band from p50 up to p95", () => { + const durations = buildOverviewSeries([ + point(0, { sessions: 1, sessionDurationP50Ms: 1_000, sessionDurationP95Ms: 4_000 }), + ]) + const spec = buildOverviewPlotSpec("sessionDuration", input({ series: durations })) + expect(spec.marks.map((mark) => mark.kind)).toEqual(["spread", "line"]) + expect(spec.rows[0]?.p95_base).toBe(1_000) + expect(spec.rows[0]?.p95).toBe(4_000) + }) +}) + +describe("overviewAxisGutter", () => { + const axis = (yMax: number, format: (value: number) => string): OverviewPlotSpec => + ({ yMax, format }) as OverviewPlotSpec + + it("prints the floor as a digit and the top through the chart's formatter", () => { + const spec = axis(0.5, (value) => `$${value.toFixed(2)}`) + expect(overviewAxisTick(spec, 0)).toBe("0") + expect(overviewAxisTick(spec, spec.yMax)).toBe("$0.50") + }) + + it("holds the design's width for the labels it was drawn around", () => { + expect(overviewAxisGutter([axis(1, () => "100%"), axis(8, () => "8.0")])).toBe(32) + }) + + it("widens to the widest label in the grid, and gives every plot the same one", () => { + const narrow = axis(1, () => "100%") + const wide = axis(150_000, () => "150.0K") + expect(overviewAxisGutter([narrow])).toBe(32) + expect(overviewAxisGutter([narrow, wide])).toBe(overviewAxisGutter([wide])) + expect(overviewAxisGutter([narrow, wide])).toBeGreaterThan(overviewAxisGutter([narrow])) + }) + + it("leaves every label room to sit right-aligned off the plot", () => { + // 6.02px per character at the 10px mono tick size, plus the tick padding. + for (const label of ["$0.50", "80.0K", "10.0%", "2m 30s", "100%"]) { + const gutter = overviewAxisGutter([axis(1, () => label)]) + expect(gutter - OVERVIEW_TICK_PADDING).toBeGreaterThanOrEqual(label.length * 6.02) + } + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-chart-specs.ts b/apps/web/src/lib/agent-sessions/overview-chart-specs.ts new file mode 100644 index 000000000..dae37ee72 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-chart-specs.ts @@ -0,0 +1,388 @@ +import { formatWarehouseDateTime } from "@maple/query-engine" +import { formatErrorRate, formatNumber, formatPercent } from "@maple/ui/lib/format" +import { formatSessionDuration } from "@maple/ui/lib/replay-format" + +import { + OVERVIEW_MODEL_MIX_OTHER, + OVERVIEW_TOKEN_BANDS, + OVERVIEW_TOKEN_FALLBACK_BAND, + formatOverviewCount, + formatPerSession, + type OverviewChartId, + type OverviewModelMix, + type OverviewSeriesPoint, + type OverviewTokenBandKey, +} from "./overview-analytics" +import { formatCost } from "./session-summary" + +/* ------------------------------------------------------------------------------------------------- + * The shape a small multiple is drawn from + * -----------------------------------------------------------------------------------------------*/ + +/** One bucket, wide: every mark on the chart reads its own field off this row. */ +export interface OverviewPlotRow extends Record { + bucket: string + date: Date +} + +/** + * `line` is the subject, `ghost` the previous period behind it, `band` a layer + * of a stack, `spread` the faint region a line of the same colour reads over. + */ +export type OverviewPlotKind = "line" | "ghost" | "band" | "spread" + +export interface OverviewPlotMark { + /** The row field this mark plots — a band's TOP edge. */ + readonly key: string + readonly color: string + readonly kind: OverviewPlotKind + /** A band's floor field. A line sits on the axis and omits it. */ + readonly base?: string + /** The tooltip's name for the series. */ + readonly label: string +} + +export interface OverviewPlotLegendItem { + readonly label: string + readonly color: string + readonly kind: OverviewPlotKind +} + +export interface OverviewPlotSpec { + readonly rows: ReadonlyArray + /** Painted in order — bands first, then the lines that read over them. */ + readonly marks: ReadonlyArray + readonly legend: ReadonlyArray + /** Bands the legend did not name, as the design's trailing `+2`. */ + readonly legendMore: number + /** The axis top. Every plot draws exactly `0` and this. */ + readonly yMax: number + readonly format: (value: number) => string +} + +export interface OverviewPlotInput { + readonly series: ReadonlyArray + /** Already shifted onto this axis; empty with the comparison off. */ + readonly previousSeries: ReadonlyArray + readonly modelMix: OverviewModelMix +} + +/* ------------------------------------------------------------------------------------------------- + * The y axis's two ticks, and the gutter they need + * -----------------------------------------------------------------------------------------------*/ + +/** The gap a tick label keeps from the plot it labels. */ +export const OVERVIEW_TICK_PADDING = 6 +/** One character of the 10px mono tick label, rounded up from the 0.6em advance + * the face actually uses — the axis is painted onto a canvas, so the gutter has + * to be decided before there is anything to measure. */ +const TICK_CHAR_WIDTH = 6.2 +/** Four characters — `100%`, `4.0%`, `10.0` — and the width the design drew. */ +const MIN_GUTTER = 32 + +/** + * A tick as the axis prints it. A duration or a cost renders zero as an em + * dash, which is right for a headline and wrong for an axis floor. + */ +export const overviewAxisTick = (spec: OverviewPlotSpec, value: number): string => + value === 0 ? "0" : spec.format(value) + +/** + * The left gutter the nine plots share, wide enough for the widest label any of + * them will print. + * + * A fixed gutter only ever fits the formatter it was written for: `100%` and + * `150.0K` are the same axis and not the same width, and a label that does not + * fit is drawn straight off the canvas's left edge. Right-aligned text overflows + * leftwards, so what is lost is the leading character — the one carrying the + * magnitude, which leaves `$0.50` reading as `0.50`. + * + * One width for the whole grid rather than one per chart: the board is read + * across as much as down, and a plot starting further right than the one beside + * it reads as a different instrument. Only the top tick is measured; the floor + * is always `0`. + */ +export function overviewAxisGutter(specs: Iterable): number { + let widest = 0 + for (const spec of specs) widest = Math.max(widest, overviewAxisTick(spec, spec.yMax).length) + return Math.max(MIN_GUTTER, Math.ceil(widest * TICK_CHAR_WIDTH) + OVERVIEW_TICK_PADDING) +} + +/* ------------------------------------------------------------------------------------------------- + * Colours + * -----------------------------------------------------------------------------------------------*/ + +const PRIMARY = "var(--primary)" +/** The previous period: present, and never mistakable for the subject. */ +const GHOST = "var(--muted-foreground)" + +/** + * The token buckets wear their own designated hues rather than `--chart-1..5` + * slots — the same five the session detail's usage bar draws, so a band here + * and a segment there are the same colour for the same tokens. + */ +const TOKEN_BAND_COLOR = { + input: "var(--chart-tok-input)", + cacheRead: "var(--chart-tok-cache-read)", + cacheWrite: "var(--chart-tok-cache-write)", + output: "var(--chart-tok-output)", + reasoning: "var(--chart-tok-reasoning)", + total: PRIMARY, +} satisfies Record + +/** Short enough that five of them fit on one legend line at this width. */ +const TOKEN_BAND_SHORT = { + input: "in", + cacheRead: "cache r", + cacheWrite: "cache w", + output: "out", + reasoning: "reason", + total: "tokens", +} satisfies Record + +const MODEL_MIX_COLORS = [ + "var(--chart-1)", + "var(--chart-2)", + "var(--chart-3)", + "var(--chart-4)", + "var(--chart-5)", +] as const +/** The folded tail is a residue, not a model — it wears no chart slot. */ +const MODEL_MIX_OTHER_COLOR = "var(--muted-foreground)" +/** Model names are long; the rest of them are counted instead of listed. */ +const MODEL_MIX_LEGEND_LIMIT = 3 + +export const overviewModelMixColor = (index: number, model: string): string => + model === OVERVIEW_MODEL_MIX_OTHER + ? MODEL_MIX_OTHER_COLOR + : (MODEL_MIX_COLORS[index % MODEL_MIX_COLORS.length] ?? MODEL_MIX_OTHER_COLOR) + +/** A band's floor field, beside the field carrying its top edge. */ +const baseKey = (key: string): string => `${key}_base` + +/* ------------------------------------------------------------------------------------------------- + * The nine specs + * -----------------------------------------------------------------------------------------------*/ + +/** + * What one small multiple draws, as data. + * + * Pure so the awkward parts — which token bands survive the fallback, where a + * stack's layers sit, what the axis tops out at — are testable without a + * rendered chart, and so the plot component stays a translation of this into + * marks rather than nine branches of chart code. + */ +export function buildOverviewPlotSpec(id: OverviewChartId, input: OverviewPlotInput): OverviewPlotSpec { + switch (id) { + case "sessions": + return lineSpec(input, (point) => point.sessions, "sessions", formatOverviewCount) + case "costPerSession": + return lineSpec(input, (point) => point.costPerSession, "$ / session", formatCost) + case "tokensPerSession": + return tokenSpec(input) + case "toolCallsPerSession": + return lineSpec(input, (point) => point.toolCallsPerSession, "calls / session", formatPerSession) + case "errorRate": + return errorRateSpec(input) + case "sessionDuration": + return durationSpec(input) + case "llmCallsPerSession": + return lineSpec(input, (point) => point.llmCallsPerSession, "calls / session", formatPerSession) + case "modelMix": + return modelMixSpec(input) + case "cacheHitRatio": + return lineSpec(input, (point) => point.cacheHitRatio, "hit ratio", formatPercent, 1) + } +} + +/** One series, plus the previous period behind it when the comparison is on. */ +function lineSpec( + input: OverviewPlotInput, + read: (point: OverviewSeriesPoint) => number, + label: string, + format: (value: number) => string, + fixedMax?: number, +): OverviewPlotSpec { + const previous = new Map(input.previousSeries.map((point) => [point.bucket, read(point)])) + const rows = input.series.map((point) => + row(point.bucket, { value: read(point), prev: previous.get(point.bucket) ?? null }), + ) + const marks: OverviewPlotMark[] = [{ key: "value", label, color: PRIMARY, kind: "line" }] + // Having previous points is not the same as having one on THIS axis: the + // ghost is matched bucket for bucket, so a legend entry for a mark that + // landed nowhere would promise a line the plot cannot draw. + if (rows.some((plotRow) => plotRow.prev !== null)) { + marks.push({ key: "prev", label: "prev", color: GHOST, kind: "ghost" }) + } + return spec(rows, marks, fixedMax ?? axisTop(rows, marks), format) +} + +/** + * Tokens per session, split five ways. + * + * The band set is decided once for the whole window rather than per bucket: an + * SDK that reports no breakdown reports none all window, and a stack that + * changed its own vocabulary mid-chart would read as a shift in usage. + */ +function tokenSpec(input: OverviewPlotInput): OverviewPlotSpec { + const split = input.series.some((point) => + OVERVIEW_TOKEN_BANDS.some((band) => point.tokenBands[band] > 0), + ) + const bands: ReadonlyArray = split + ? OVERVIEW_TOKEN_BANDS + : [OVERVIEW_TOKEN_FALLBACK_BAND] + + const rows = input.series.map((point) => + stackRow( + point.bucket, + bands.map((band) => ({ key: band, value: point.tokenBands[band] })), + ), + ) + const marks = bands.map((band) => bandMark(band, TOKEN_BAND_COLOR[band], TOKEN_BAND_SHORT[band])) + return spec(rows, marks, axisTop(rows, marks), formatNumber) +} + +/** The three layers a session can fail at, on one rate axis. */ +function errorRateSpec(input: OverviewPlotInput): OverviewPlotSpec { + const rows = input.series.map((point) => + row(point.bucket, { + sessions: point.sessionErrorRate, + llm: point.llmErrorRate, + tool: point.toolErrorRate, + }), + ) + const marks: ReadonlyArray = [ + { key: "sessions", label: "sessions", color: "var(--severity-error)", kind: "line" }, + { key: "llm", label: "llm calls", color: "var(--chart-2)", kind: "line" }, + { key: "tool", label: "tool calls", color: "var(--chart-5)", kind: "line" }, + ] + return spec(rows, marks, axisTop(rows, marks), formatErrorRate) +} + +/** The median with the spread above it — the tail is the question. */ +function durationSpec(input: OverviewPlotInput): OverviewPlotSpec { + const rows = input.series.map((point) => + row(point.bucket, { + p50: point.sessionP50Ms, + p95: point.sessionP95Ms, + p95_base: point.sessionP50Ms, + }), + ) + const marks: ReadonlyArray = [ + { key: "p95", base: "p95_base", label: "p50 – p95", color: PRIMARY, kind: "spread" }, + { key: "p50", label: "p50", color: PRIMARY, kind: "line" }, + ] + return spec(rows, marks, axisTop(rows, marks), formatSessionDuration) +} + +/** Every bucket normalised to 1, so the question is share and not volume. */ +function modelMixSpec(input: OverviewPlotInput): OverviewPlotSpec { + const { models, points } = input.modelMix + const rows = points.map((point) => + stackRow( + point.bucket, + models.map((model) => ({ key: model, value: point.shares[model] ?? 0 })), + ), + ) + const marks = models.map((model, index) => bandMark(model, overviewModelMixColor(index, model), model)) + const legend = marks.slice(0, MODEL_MIX_LEGEND_LIMIT).map((mark, index) => ({ + label: `${models[index] ?? ""} ${formatPercent(modelShare(input.modelMix, models[index] ?? ""))}`, + color: mark.color, + kind: mark.kind, + })) + return { + rows, + marks, + legend, + legendMore: Math.max(0, marks.length - legend.length), + yMax: 1, + format: formatPercent, + } +} + +/** A model's share of the window's plotted spans, for its legend entry. */ +function modelShare(mix: OverviewModelMix, model: string): number { + let total = 0 + let own = 0 + for (const point of mix.points) { + for (const band of mix.models) total += point.spans[band] ?? 0 + own += point.spans[model] ?? 0 + } + return total === 0 ? 0 : own / total +} + +/* ------------------------------------------------------------------------------------------------- + * Row and axis plumbing + * -----------------------------------------------------------------------------------------------*/ + +function row(bucketMs: number, values: Record): OverviewPlotRow { + return { bucket: formatWarehouseDateTime(bucketMs), date: new Date(bucketMs), ...values } +} + +/** A stack's layers, each carrying the top edge it landed on and its floor. */ +function stackRow(bucketMs: number, layers: ReadonlyArray<{ key: string; value: number }>): OverviewPlotRow { + const values: Record = {} + let floor = 0 + for (const layer of layers) { + values[baseKey(layer.key)] = floor + floor += layer.value + values[layer.key] = floor + } + return row(bucketMs, values) +} + +const bandMark = (key: string, color: string, label: string): OverviewPlotMark => ({ + key, + base: baseKey(key), + color, + kind: "band", + label, +}) + +function spec( + rows: ReadonlyArray, + marks: ReadonlyArray, + yMax: number, + format: (value: number) => string, +): OverviewPlotSpec { + return { + rows, + marks, + legend: marks.map((mark) => ({ label: mark.label, color: mark.color, kind: mark.kind })), + legendMore: 0, + yMax, + format, + } +} + +/** + * A finer ladder than a chart library's: these plots are 86px tall, so a top + * two steps above the data spends a quarter of the height on empty air. Every + * rung still prints as a round number through the chart's own formatter. + */ +const CEILING_STEPS = [1, 1.2, 1.5, 1.8, 2, 2.5, 3, 4, 5, 6, 8, 10] + +/** + * The axis top: the next round number above the largest plotted value. + * + * Above, and never equal to — a flat series at exactly the top rides the + * ceiling and reads as clipped rather than as steady, which is the one thing + * these nine charts exist to show, so a maximum landing on a rung takes the + * rung after it. A stack's layers carry their cumulative top, so the same + * maximum covers both shapes; an all-zero window still gets a `1` so the scale + * has a domain, which draws the flat floor it should. + */ +function axisTop(rows: ReadonlyArray, marks: ReadonlyArray): number { + let max = 0 + for (const plotRow of rows) { + for (const mark of marks) { + const value = plotRow[mark.key] + if (typeof value === "number" && value > max) max = value + } + } + if (max <= 0) return 1 + const magnitude = 10 ** Math.floor(Math.log10(max)) + const scaled = max / magnitude + const step = CEILING_STEPS.find((candidate) => candidate > scaled * 1.000_001) ?? 10 + return step * magnitude +} diff --git a/apps/web/src/lib/agent-sessions/overview-search.test.ts b/apps/web/src/lib/agent-sessions/overview-search.test.ts new file mode 100644 index 000000000..fc392a628 --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-search.test.ts @@ -0,0 +1,158 @@ +import { describe, expect, it } from "vitest" + +import { + OVERVIEW_DIMENSIONS, + activeOverviewFilters, + clearOverviewFilters, + compareEnabled, + failingOnly, + overviewApiDimension, + overviewFilterPatch, + overviewWindowLabel, + sessionsLinkSearch, + toggleOverviewFilter, + type AgentOverviewSearch, +} from "./overview-search" + +describe("overviewApiDimension", () => { + it("renames the page's framework to the warehouse's vendor and leaves the rest", () => { + expect(overviewApiDimension("framework")).toBe("vendor") + expect(overviewApiDimension("model")).toBe("model") + expect(overviewApiDimension("tool")).toBe("tool") + }) +}) + +describe("compareEnabled", () => { + it("is on when the URL says nothing, and only `false` turns it off", () => { + expect(compareEnabled({})).toBe(true) + expect(compareEnabled({ compare: true })).toBe(true) + expect(compareEnabled({ compare: false })).toBe(false) + }) +}) + +describe("failingOnly", () => { + it("reads only an explicit true", () => { + expect(failingOnly({})).toBe(false) + expect(failingOnly({ hasErrors: false })).toBe(false) + expect(failingOnly({ hasErrors: true })).toBe(true) + }) +}) + +describe("activeOverviewFilters", () => { + it("lists the set dimensions in the dimensions' own order", () => { + const search: AgentOverviewSearch = { tool: "run_tests", model: "opus", environment: "prd" } + expect(activeOverviewFilters(search)).toEqual([ + { dimension: "model", value: "opus" }, + { dimension: "environment", value: "prd" }, + { dimension: "tool", value: "run_tests" }, + ]) + }) + + it("is empty when only the toggles are set", () => { + expect(activeOverviewFilters({ hasErrors: true, compare: false })).toEqual([]) + }) +}) + +describe("clearOverviewFilters", () => { + it("clears every dimension and leaves the toggles alone", () => { + const patch = clearOverviewFilters() + expect(patch).toEqual({ + model: undefined, + agent: undefined, + service: undefined, + framework: undefined, + environment: undefined, + tool: undefined, + }) + expect("hasErrors" in patch).toBe(false) + expect("compare" in patch).toBe(false) + }) +}) + +describe("toggleOverviewFilter", () => { + it("selects a key that is not the current one", () => { + expect(toggleOverviewFilter({}, "model", "opus")).toEqual({ model: "opus" }) + }) + + it("clears the dimension when the key is already selected", () => { + expect(toggleOverviewFilter({ model: "opus" }, "model", "opus")).toEqual({ model: undefined }) + }) + + it("clears rather than selects the unattributed key, which has no spelling", () => { + expect(toggleOverviewFilter({ agent: "a" }, "agent", "")).toEqual({ agent: undefined }) + }) +}) + +describe("overviewFilterPatch", () => { + // Written out rather than built from a computed key, so every dimension has + // to be covered by hand — a missing arm is a control that silently does + // nothing. + it("sets exactly its own dimension, for all six", () => { + expect(overviewFilterPatch("model", "claude-opus-5")).toEqual({ model: "claude-opus-5" }) + expect(overviewFilterPatch("agent", "captain")).toEqual({ agent: "captain" }) + expect(overviewFilterPatch("service", "api")).toEqual({ service: "api" }) + expect(overviewFilterPatch("framework", "eve")).toEqual({ framework: "eve" }) + expect(overviewFilterPatch("environment", "prd")).toEqual({ environment: "prd" }) + expect(overviewFilterPatch("tool", "run_tests")).toEqual({ tool: "run_tests" }) + }) + + it("clears its own dimension and no other", () => { + for (const dimension of OVERVIEW_DIMENSIONS) { + expect(overviewFilterPatch(dimension, undefined)).toEqual({ [dimension]: undefined }) + } + }) +}) + +describe("sessionsLinkSearch", () => { + it("widens each single value into the list's array-valued key", () => { + expect( + sessionsLinkSearch({ + framework: "eve", + model: "opus", + agent: "captain", + service: "api", + environment: "prd", + tool: "run_tests", + }), + ).toEqual({ + vendors: ["eve"], + services: ["api"], + environments: ["prd"], + models: ["opus"], + agents: ["captain"], + tools: ["run_tests"], + hasErrors: undefined, + }) + }) + + it("carries the board's failing-only toggle", () => { + expect(sessionsLinkSearch({ hasErrors: true }).hasErrors).toBe(true) + }) + + it("lets the errored tab ask for failures the board is not filtered to", () => { + expect(sessionsLinkSearch({}, { hasErrors: true }).hasErrors).toBe(true) + }) +}) + +describe("overviewWindowLabel", () => { + const hours = (count: number) => count * 3_600_000 + /** An absolute range in the URL, which is what makes the default irrelevant. */ + const ABSOLUTE = { startTime: "2026-09-10 00:00:00", endTime: "2026-09-10 03:00:00" } + + it("lets a preset name itself", () => { + expect(overviewWindowLabel({ timePreset: "24h" }, hours(24))).toBe("24h") + }) + + it("falls back to the page's default only while the URL carries no window", () => { + expect(overviewWindowLabel({}, hours(3))).toBe("7d") + // Half a range is not a range: the resolver would still use the default. + expect(overviewWindowLabel({ startTime: ABSOLUTE.startTime }, hours(3))).toBe("7d") + }) + + it("names an absolute range after its own length, not after the default", () => { + expect(overviewWindowLabel(ABSOLUTE, hours(3))).toBe("3h") + expect(overviewWindowLabel(ABSOLUTE, hours(24))).toBe("24h") + expect(overviewWindowLabel(ABSOLUTE, hours(72))).toBe("3d") + expect(overviewWindowLabel(ABSOLUTE, 45 * 60_000)).toBe("45m") + }) +}) diff --git a/apps/web/src/lib/agent-sessions/overview-search.ts b/apps/web/src/lib/agent-sessions/overview-search.ts new file mode 100644 index 000000000..39ed993de --- /dev/null +++ b/apps/web/src/lib/agent-sessions/overview-search.ts @@ -0,0 +1,223 @@ +// The URL is the whole state of `/agent-sessions/overview`. Every control — +// the six dimension selects, the two toggles, a breakdown row, a mover line — +// writes a search param and reads it back, so a link carries exactly the board +// someone was looking at and Back undoes one decision at a time. +// +// Declared here rather than in the route file because the hook, the view and +// the lab all need the decoded shape, and only the route needs the schema. + +import { Schema } from "effect" + +import type { AiOverviewDimension } from "@maple/domain/http" +import type { TimeRangeSearch } from "@/components/time-range-picker/search" +import { BooleanFromStringParam } from "@/lib/search-params" + +const BooleanParam = Schema.optional(Schema.Union([Schema.Boolean, BooleanFromStringParam])) + +/** + * The six dimensions the page groups and filters by, in the order the + * breakdown tabs show them. + * + * `framework` is the page's word for what the warehouse calls a vendor — the + * SDK or gateway that produced the spans. The URL key and the dimension id are + * the same string on purpose; {@link overviewApiDimension} is the one place + * the rename happens. + */ +export const OVERVIEW_DIMENSIONS = ["model", "agent", "service", "framework", "environment", "tool"] as const +export type OverviewDimension = (typeof OVERVIEW_DIMENSIONS)[number] + +/** The dimension as the breakdown endpoint spells it. */ +export function overviewApiDimension(dimension: OverviewDimension): AiOverviewDimension { + return dimension === "framework" ? "vendor" : dimension +} + +/** + * Spread into the route's own `Schema.Struct`, ahead of `TimeRangeSearchFields`. + * + * `Schema.optional` throughout (not `optionalKey`) for the reason the + * time-range fields give: TanStack Router hands back keys that are + * present-but-`undefined`, and clearing a filter writes `undefined` explicitly. + * + * One value per dimension, not an array: this page is read by narrowing to one + * thing at a time, and a row click that appended to a set would need a second + * gesture to mean "only this". + */ +export const OverviewSearchFields = { + /** The SDK or gateway, as the gateway stamps it (e.g. `eve`), not a label. */ + framework: Schema.optional(Schema.String), + model: Schema.optional(Schema.String), + agent: Schema.optional(Schema.String), + service: Schema.optional(Schema.String), + environment: Schema.optional(Schema.String), + tool: Schema.optional(Schema.String), + /** Sessions with at least one failed span. */ + hasErrors: BooleanParam, + /** The previous-period comparison. On unless the URL says `false`. */ + compare: BooleanParam, +} + +export const AgentOverviewSearch = Schema.Struct(OverviewSearchFields) +export type AgentOverviewSearch = Schema.Schema.Type + +/** Wide enough that a nightly agent shows up at all. */ +export const AGENT_OVERVIEW_DEFAULT_PRESET = "7d" + +/** + * What the page calls the window it is showing — `7d`, `24h`, `45m`. + * + * A preset names itself. An absolute range has no preset, and the default above + * does not name it either: the resolver hands a start/end pair straight back + * and never looks at the default, so a two-hour range picked by hand would + * otherwise be labelled "prev 7d". It is named after its own length instead, to + * the nearest whole unit. + */ +export function overviewWindowLabel(search: TimeRangeSearch, windowMs: number): string { + if (search.timePreset !== undefined) return search.timePreset + if (search.startTime === undefined || search.endTime === undefined) { + return AGENT_OVERVIEW_DEFAULT_PRESET + } + const minutes = Math.max(1, Math.round(windowMs / 60_000)) + if (minutes < 60) return `${minutes}m` + const hours = Math.round(minutes / 60) + return hours < 48 ? `${hours}h` : `${Math.round(hours / 24)}d` +} + +/** The value in force for one dimension, or nothing. */ +export const selectedDimensionValue = ( + search: AgentOverviewSearch, + dimension: OverviewDimension, +): string | undefined => search[dimension] + +/** The comparison is on by default, so only `compare=false` turns it off. */ +export const compareEnabled = (search: AgentOverviewSearch): boolean => search.compare !== false + +export const failingOnly = (search: AgentOverviewSearch): boolean => search.hasErrors === true + +/** One option in a dimension select, with the sessions behind it. */ +export interface OverviewFacetOption { + readonly name: string + readonly count: number +} + +/** The window's facet values per dimension, unfiltered — picking one model must + * not erase the others from the select. */ +export type OverviewFacets = Record> + +export const EMPTY_OVERVIEW_FACETS = { + model: [], + agent: [], + service: [], + framework: [], + environment: [], + tool: [], +} satisfies OverviewFacets + +export interface OverviewFilterChip { + readonly dimension: OverviewDimension + readonly value: string +} + +/** The active dimension filters, in the dimensions' own order — the scope row. */ +export function activeOverviewFilters(search: AgentOverviewSearch): ReadonlyArray { + return OVERVIEW_DIMENSIONS.flatMap((dimension) => { + const value = search[dimension] + return value === undefined ? [] : [{ dimension, value }] + }) +} + +/** The patch "Clear all" applies: every dimension filter off, the toggles kept. */ +export function clearOverviewFilters(): Partial { + return { + model: undefined, + agent: undefined, + service: undefined, + framework: undefined, + environment: undefined, + tool: undefined, + } +} + +/** + * The patch a breakdown row or a mover line applies: pick this key, or clear + * the dimension when it is already the selected one. + * + * `''` is a real breakdown key — a span that carries no value for the dimension + * — but the selection contract has no spelling for "the unnamed one", so a row + * under it clears the dimension rather than selecting nothing. + */ +export function toggleOverviewFilter( + search: AgentOverviewSearch, + dimension: OverviewDimension, + key: string, +): Partial { + return overviewFilterPatch(dimension, key === "" || search[dimension] === key ? undefined : key) +} + +/** + * The patch that sets one dimension. + * + * Written out rather than built from a computed key: a computed key widens the + * patch to an open dictionary, and the whole point of the patch type is that + * only a real search field can reach the URL. + */ +export function overviewFilterPatch( + dimension: OverviewDimension, + value: string | undefined, +): Partial { + switch (dimension) { + case "model": + return { model: value } + case "agent": + return { agent: value } + case "service": + return { service: value } + case "framework": + return { framework: value } + case "environment": + return { environment: value } + case "tool": + return { tool: value } + } +} + +/** + * The Sessions list's own search params, as a link into it has to spell them. + * + * Mutable arrays rather than the readonly ones the list reads its state + * through: the route declares its array params with `Schema.mutable`, and a + * `` is checked against the route's search input. + */ +export interface AgentSessionsLinkSearch { + vendors?: string[] + services?: string[] + environments?: string[] + models?: string[] + agents?: string[] + tools?: string[] + hasErrors?: boolean +} + +/** + * What travels from this page into the Sessions list. + * + * The list filters by the same six dimensions under array-valued keys, so a + * single value becomes a one-element array. The window does NOT travel: the + * list has no picker and reads a rolling week of its own, and a window param it + * does not validate is dropped by the router rather than honoured. + */ +export function sessionsLinkSearch( + search: AgentOverviewSearch, + options?: { hasErrors?: boolean }, +): AgentSessionsLinkSearch { + const one = (value: string | undefined) => (value === undefined ? undefined : [value]) + const errors = options?.hasErrors ?? search.hasErrors === true + return { + vendors: one(search.framework), + services: one(search.service), + environments: one(search.environment), + models: one(search.model), + agents: one(search.agent), + tools: one(search.tool), + hasErrors: errors ? true : undefined, + } +} diff --git a/apps/web/src/lib/agent-sessions/use-agent-overview.test.ts b/apps/web/src/lib/agent-sessions/use-agent-overview.test.ts new file mode 100644 index 000000000..989aaa88e --- /dev/null +++ b/apps/web/src/lib/agent-sessions/use-agent-overview.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest" + +import { overviewSelection, overviewSessionsInput } from "./use-agent-overview" + +const WINDOW = { startTime: "2026-09-04 00:00:00", endTime: "2026-09-11 00:00:00" } + +describe("overviewSelection", () => { + it("sends the window and nothing else for an untouched page", () => { + expect(overviewSelection({}, WINDOW)).toEqual({ + ...WINDOW, + framework: undefined, + model: undefined, + agent: undefined, + service: undefined, + environment: undefined, + tool: undefined, + hasErrors: undefined, + }) + }) + + it("puts every dimension filter in the selection, so each is in the cache key", () => { + const selection = overviewSelection( + { + framework: "eve", + model: "claude-opus-5", + agent: "release-captain", + service: "api", + environment: "production", + tool: "run_tests", + }, + WINDOW, + ) + expect(selection.framework).toBe("eve") + expect(selection.model).toBe("claude-opus-5") + expect(selection.agent).toBe("release-captain") + expect(selection.service).toBe("api") + expect(selection.environment).toBe("production") + expect(selection.tool).toBe("run_tests") + }) + + it("drops an explicitly false failing-only rather than sending it", () => { + expect(overviewSelection({ hasErrors: false }, WINDOW).hasErrors).toBeUndefined() + expect(overviewSelection({ hasErrors: true }, WINDOW).hasErrors).toBe(true) + }) + + it("leaves the comparison out — it is a client-side reading of one read", () => { + expect(overviewSelection({ compare: false }, WINDOW)).toEqual(overviewSelection({}, WINDOW)) + }) +}) + +describe("overviewSessionsInput", () => { + it("widens each single value into the list endpoint's array key", () => { + const input = overviewSessionsInput({ model: "claude-opus-5", framework: "eve" }, WINDOW, { + sortBy: "cost", + }) + expect(input.models).toEqual(["claude-opus-5"]) + expect(input.vendorIds).toEqual(["eve"]) + expect(input.agentNames).toBeUndefined() + }) + + it("asks for one short page, worst first", () => { + const input = overviewSessionsInput({}, WINDOW, { sortBy: "durationMs" }) + expect(input.sortBy).toBe("durationMs") + expect(input.sortDir).toBe("desc") + expect(input.limit).toBe(6) + }) + + it("lets the errored tab ask for failures the board is not filtered to", () => { + expect( + overviewSessionsInput({}, WINDOW, { sortBy: "errorSpanCount", hasErrors: true }).hasErrors, + ).toBe(true) + }) +}) diff --git a/apps/web/src/lib/agent-sessions/use-agent-overview.ts b/apps/web/src/lib/agent-sessions/use-agent-overview.ts new file mode 100644 index 000000000..0c0c99b7d --- /dev/null +++ b/apps/web/src/lib/agent-sessions/use-agent-overview.ts @@ -0,0 +1,192 @@ +// Every warehouse read `/agent-sessions/overview` makes, behind one hook. +// +// The page itself never touches an atom: it takes the `Result`s this returns +// and renders them. That is what lets the lab mount the same view over +// fixtures, and it keeps the wire shape confined to the mappers in +// `api/warehouse/ai-agent-overview.ts`. +// +// Nine reads, because the board is nine questions: the summary, one breakdown +// per dimension (those tabs are component state and the movers rail reads all +// six anyway), the model mix, and ONE page of six sessions — the Top sessions +// tab the reader is actually on. The other two are read when they are opened, +// and the atom family holds them from then on. + +import { useMemo } from "react" + +import type { Effect } from "effect" +import type { AiSessionSortKey } from "@maple/domain/http" + +import type { + AiOverviewBreakdownData, + AiOverviewModelMixData, + AiOverviewSelection, + AiOverviewSummaryData, +} from "@/api/warehouse/ai-agent-overview" +import type { ListAiSessionsInput, listAiSessions } from "@/api/warehouse/ai-sessions" +import { useRefreshableAtomValue } from "@/hooks/use-refreshable-atom-value" +import type { Result } from "@/lib/effect-atom" +import type { QueryAtomFailure } from "@/lib/services/atoms/warehouse-query-atoms" +import { + aiOverviewBreakdownResultAtom, + aiOverviewModelMixResultAtom, + aiOverviewSummaryResultAtom, + listAiSessionsResultAtom, +} from "@/lib/services/atoms/warehouse-query-atoms" + +import { smallMultipleBucketSeconds } from "./overview-buckets" +import { overviewApiDimension, type AgentOverviewSearch, type OverviewDimension } from "./overview-search" + +export interface AgentOverviewWindow { + readonly startTime: string + readonly endTime: string +} + +/** Enough rows to recognise a pattern, few enough to read without scrolling. */ +export const OVERVIEW_TOP_SESSIONS_LIMIT = 6 + +/** The three readings of "show me the sessions behind this". */ +export const OVERVIEW_TOP_SESSION_TABS = ["cost", "duration", "errored"] as const +export type OverviewTopSessionTab = (typeof OVERVIEW_TOP_SESSION_TABS)[number] + +/** How each tab asks the list for its six rows. */ +const TOP_SESSION_READS = { + cost: { sortBy: "cost" }, + duration: { sortBy: "durationMs" }, + errored: { sortBy: "errorSpanCount", hasErrors: true }, +} satisfies Record + +/** The list read's own page shape — the rows the Top sessions table renders. */ +export type AgentOverviewSessionsPage = Effect.Success> +type SessionsResult = Result.Result + +export interface AgentOverviewResults { + readonly summary: Result.Result + /** In `OVERVIEW_DIMENSIONS` order. */ + readonly breakdowns: ReadonlyArray<{ + readonly dimension: OverviewDimension + readonly result: Result.Result + }> + readonly modelMix: Result.Result + /** The active tab's page. The tab is the caller's state, so switching it is + * what issues the other reads. */ + readonly topSessions: SessionsResult +} + +/** + * The search params as the three overview endpoints take them. + * + * All six filters are server-side, so every read re-scopes to the toolbar and + * the tiles, the grid and the tables always describe the same sessions. That + * also puts every filter in every atom's cache key, which is what makes a + * chosen model a new read rather than a stale one. + */ +export function overviewSelection( + search: AgentOverviewSearch, + window: AgentOverviewWindow, +): AiOverviewSelection { + return { + startTime: window.startTime, + endTime: window.endTime, + framework: search.framework, + model: search.model, + agent: search.agent, + service: search.service, + environment: search.environment, + tool: search.tool, + hasErrors: search.hasErrors === true ? true : undefined, + } +} + +/** + * One page of the sessions list, under the board's own scope. + * + * The list is the concrete end of every number above it, so it must filter by + * exactly the same six dimensions — under the array-valued names the list + * endpoint uses. + */ +export function overviewSessionsInput( + search: AgentOverviewSearch, + window: AgentOverviewWindow, + options: { sortBy: AiSessionSortKey; hasErrors?: boolean }, +): ListAiSessionsInput { + const one = (value: string | undefined) => (value === undefined ? undefined : [value]) + const errors = options.hasErrors ?? search.hasErrors === true + return { + startTime: window.startTime, + endTime: window.endTime, + vendorIds: one(search.framework), + serviceNames: one(search.service), + deploymentEnvs: one(search.environment), + models: one(search.model), + agentNames: one(search.agent), + toolNames: one(search.tool), + hasErrors: errors ? true : undefined, + sortBy: options.sortBy, + sortDir: "desc", + limit: OVERVIEW_TOP_SESSIONS_LIMIT, + } +} + +export function useAgentOverview( + search: AgentOverviewSearch, + window: AgentOverviewWindow, + topSessionTab: OverviewTopSessionTab, +): AgentOverviewResults { + const selection = useMemo(() => overviewSelection(search, window), [search, window]) + // The grid is nine ~104px-tall plots rather than one wide chart, so the + // buckets are cut at a width that reads at that size. + const bucketSeconds = smallMultipleBucketSeconds(window.startTime, window.endTime) + const bucketed = { ...selection, bucketSeconds } + + const summary = useRefreshableAtomValue(aiOverviewSummaryResultAtom({ data: bucketed })) + const modelMix = useRefreshableAtomValue(aiOverviewModelMixResultAtom({ data: bucketed })) + + // One call per dimension, written out: a loop over the dimensions would be a + // hook in a loop. + const model = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "model" } }), + ) + const agent = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "agent" } }), + ) + const service = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "service" } }), + ) + const framework = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ + data: { ...selection, dimension: overviewApiDimension("framework") }, + }), + ) + const environment = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "environment" } }), + ) + const tool = useRefreshableAtomValue( + aiOverviewBreakdownResultAtom({ data: { ...selection, dimension: "tool" } }), + ) + + const topSessions = useRefreshableAtomValue( + listAiSessionsResultAtom({ + data: overviewSessionsInput(search, window, TOP_SESSION_READS[topSessionTab]), + }), + ) + + // Held across renders: the page builds its whole view model from these, and + // a fresh array of the same six results every render would rebuild it — + // along with every chart memo keyed on the series it produces. + return useMemo( + () => ({ + summary, + modelMix, + breakdowns: [ + { dimension: "model", result: model }, + { dimension: "agent", result: agent }, + { dimension: "service", result: service }, + { dimension: "framework", result: framework }, + { dimension: "environment", result: environment }, + { dimension: "tool", result: tool }, + ], + topSessions, + }), + [summary, modelMix, model, agent, service, framework, environment, tool, topSessions], + ) +} diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index c6eb0b65c..a3c9f5abf 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -115,6 +115,11 @@ import { listReplays, } from "@/api/warehouse/replays" import { getAiSessionSpans, getAiSessionSummary, getAiSessionsFacets, listAiSessions } from "@/api/warehouse/ai-sessions" +import { + getAiOverviewBreakdown, + getAiOverviewModelMix, + getAiOverviewSummary, +} from "@/api/warehouse/ai-agent-overview" import { getWebAnalyticsBreakdowns, getWebAnalyticsEvents, @@ -350,6 +355,22 @@ export const aiSessionSummaryResultAtom = makeQueryAtomFamily(getAiSessionSummar staleTime: 60_000, }) +// The overview board's three reads. 30s like every other filtered analytics +// atom: the whole input is the cache key, so each of the six breakdown +// dimensions keys separately and a filter change is a new read rather than a +// stale one. +export const aiOverviewSummaryResultAtom = makeQueryAtomFamily(getAiOverviewSummary, { + staleTime: 30_000, +}) + +export const aiOverviewBreakdownResultAtom = makeQueryAtomFamily(getAiOverviewBreakdown, { + staleTime: 30_000, +}) + +export const aiOverviewModelMixResultAtom = makeQueryAtomFamily(getAiOverviewModelMix, { + staleTime: 30_000, +}) + export const replaysFacetsResultAtom = makeQueryAtomFamily(getReplaysFacets, { staleTime: 30_000, }) diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index f91b3da1f..b36a7f901 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -28,6 +28,7 @@ import { Route as SignInRouteImport } from './routes/sign-in' import { Route as SignUpRouteImport } from './routes/sign-up' import { Route as AgentSessionsIndexRouteImport } from './routes/agent-sessions/index' import { Route as AgentSessionsSessionIdRouteImport } from './routes/agent-sessions/$sessionId' +import { Route as AgentSessionsOverviewRouteImport } from './routes/agent-sessions/overview' import { Route as AlertsIndexRouteImport } from './routes/alerts/index' import { Route as AlertsRuleIdRouteImport } from './routes/alerts/$ruleId' import { Route as AlertsCreateRouteImport } from './routes/alerts/create' @@ -44,6 +45,7 @@ import { Route as InfraDiscoverRouteImport } from './routes/infra/discover' import { Route as InvestigationsIndexRouteImport } from './routes/investigations/index' import { Route as InvestigationsIdRouteImport } from './routes/investigations/$id' import { Route as LabIndexRouteImport } from './routes/lab/index' +import { Route as LabAgentOverviewRouteImport } from './routes/lab/agent-overview' import { Route as LabAgentSessionRouteImport } from './routes/lab/agent-session' import { Route as LabAgentSessionsRouteImport } from './routes/lab/agent-sessions' import { Route as LabChartsRouteImport } from './routes/lab/charts' @@ -194,6 +196,11 @@ const AgentSessionsSessionIdRoute = AgentSessionsSessionIdRouteImport.update({ path: '/agent-sessions/$sessionId', getParentRoute: () => rootRouteImport, } as any) +const AgentSessionsOverviewRoute = AgentSessionsOverviewRouteImport.update({ + id: '/agent-sessions/overview', + path: '/agent-sessions/overview', + getParentRoute: () => rootRouteImport, +} as any) const AlertsIndexRoute = AlertsIndexRouteImport.update({ id: '/alerts/', path: '/alerts/', @@ -274,6 +281,11 @@ const LabIndexRoute = LabIndexRouteImport.update({ path: '/', getParentRoute: () => LabRouteRoute, } as any) +const LabAgentOverviewRoute = LabAgentOverviewRouteImport.update({ + id: '/agent-overview', + path: '/agent-overview', + getParentRoute: () => LabRouteRoute, +} as any) const LabAgentSessionRoute = LabAgentSessionRouteImport.update({ id: '/agent-session', path: '/agent-session', @@ -576,6 +588,7 @@ export interface FileRoutesByFullPath { '/sign-in': typeof SignInRoute '/sign-up': typeof SignUpRoute '/agent-sessions/$sessionId': typeof AgentSessionsSessionIdRoute + '/agent-sessions/overview': typeof AgentSessionsOverviewRoute '/alerts/$ruleId': typeof AlertsRuleIdRoute '/alerts/create': typeof AlertsCreateRoute '/anomalies/$incidentId': typeof AnomaliesIncidentIdRoute @@ -584,6 +597,7 @@ export interface FileRoutesByFullPath { '/infra/$hostName': typeof InfraHostNameRoute '/infra/discover': typeof InfraDiscoverRoute '/investigations/$id': typeof InvestigationsIdRoute + '/lab/agent-overview': typeof LabAgentOverviewRoute '/lab/agent-session': typeof LabAgentSessionRoute '/lab/agent-sessions': typeof LabAgentSessionsRoute '/lab/charts': typeof LabChartsRoute @@ -666,6 +680,7 @@ export interface FileRoutesByTo { '/sign-in': typeof SignInRoute '/sign-up': typeof SignUpRoute '/agent-sessions/$sessionId': typeof AgentSessionsSessionIdRoute + '/agent-sessions/overview': typeof AgentSessionsOverviewRoute '/alerts/$ruleId': typeof AlertsRuleIdRoute '/alerts/create': typeof AlertsCreateRoute '/anomalies/$incidentId': typeof AnomaliesIncidentIdRoute @@ -674,6 +689,7 @@ export interface FileRoutesByTo { '/infra/$hostName': typeof InfraHostNameRoute '/infra/discover': typeof InfraDiscoverRoute '/investigations/$id': typeof InvestigationsIdRoute + '/lab/agent-overview': typeof LabAgentOverviewRoute '/lab/agent-session': typeof LabAgentSessionRoute '/lab/agent-sessions': typeof LabAgentSessionsRoute '/lab/charts': typeof LabChartsRoute @@ -758,6 +774,7 @@ export interface FileRoutesById { '/sign-in': typeof SignInRoute '/sign-up': typeof SignUpRoute '/agent-sessions/$sessionId': typeof AgentSessionsSessionIdRoute + '/agent-sessions/overview': typeof AgentSessionsOverviewRoute '/alerts/$ruleId': typeof AlertsRuleIdRoute '/alerts/create': typeof AlertsCreateRoute '/anomalies/$incidentId': typeof AnomaliesIncidentIdRoute @@ -766,6 +783,7 @@ export interface FileRoutesById { '/infra/$hostName': typeof InfraHostNameRoute '/infra/discover': typeof InfraDiscoverRoute '/investigations/$id': typeof InvestigationsIdRoute + '/lab/agent-overview': typeof LabAgentOverviewRoute '/lab/agent-session': typeof LabAgentSessionRoute '/lab/agent-sessions': typeof LabAgentSessionsRoute '/lab/charts': typeof LabChartsRoute @@ -851,6 +869,7 @@ export interface FileRouteTypes { | '/sign-in' | '/sign-up' | '/agent-sessions/$sessionId' + | '/agent-sessions/overview' | '/alerts/$ruleId' | '/alerts/create' | '/anomalies/$incidentId' @@ -859,6 +878,7 @@ export interface FileRouteTypes { | '/infra/$hostName' | '/infra/discover' | '/investigations/$id' + | '/lab/agent-overview' | '/lab/agent-session' | '/lab/agent-sessions' | '/lab/charts' @@ -941,6 +961,7 @@ export interface FileRouteTypes { | '/sign-in' | '/sign-up' | '/agent-sessions/$sessionId' + | '/agent-sessions/overview' | '/alerts/$ruleId' | '/alerts/create' | '/anomalies/$incidentId' @@ -949,6 +970,7 @@ export interface FileRouteTypes { | '/infra/$hostName' | '/infra/discover' | '/investigations/$id' + | '/lab/agent-overview' | '/lab/agent-session' | '/lab/agent-sessions' | '/lab/charts' @@ -1032,6 +1054,7 @@ export interface FileRouteTypes { | '/sign-in' | '/sign-up' | '/agent-sessions/$sessionId' + | '/agent-sessions/overview' | '/alerts/$ruleId' | '/alerts/create' | '/anomalies/$incidentId' @@ -1040,6 +1063,7 @@ export interface FileRouteTypes { | '/infra/$hostName' | '/infra/discover' | '/investigations/$id' + | '/lab/agent-overview' | '/lab/agent-session' | '/lab/agent-sessions' | '/lab/charts' @@ -1124,6 +1148,7 @@ export interface RootRouteChildren { SignInRoute: typeof SignInRoute SignUpRoute: typeof SignUpRoute AgentSessionsSessionIdRoute: typeof AgentSessionsSessionIdRoute + AgentSessionsOverviewRoute: typeof AgentSessionsOverviewRoute AlertsRuleIdRoute: typeof AlertsRuleIdRoute AlertsCreateRoute: typeof AlertsCreateRoute AnomaliesIncidentIdRoute: typeof AnomaliesIncidentIdRoute @@ -1310,6 +1335,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AgentSessionsSessionIdRouteImport parentRoute: typeof rootRouteImport } + '/agent-sessions/overview': { + id: '/agent-sessions/overview' + path: '/agent-sessions/overview' + fullPath: '/agent-sessions/overview' + preLoaderRoute: typeof AgentSessionsOverviewRouteImport + parentRoute: typeof rootRouteImport + } '/alerts/': { id: '/alerts/' path: '/alerts' @@ -1422,6 +1454,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof LabIndexRouteImport parentRoute: typeof LabRouteRoute } + '/lab/agent-overview': { + id: '/lab/agent-overview' + path: '/agent-overview' + fullPath: '/lab/agent-overview' + preLoaderRoute: typeof LabAgentOverviewRouteImport + parentRoute: typeof LabRouteRoute + } '/lab/agent-session': { id: '/lab/agent-session' path: '/agent-session' @@ -1804,6 +1843,7 @@ declare module '@tanstack/react-router' { } interface LabRouteRouteChildren { + LabAgentOverviewRoute: typeof LabAgentOverviewRoute LabAgentSessionRoute: typeof LabAgentSessionRoute LabAgentSessionsRoute: typeof LabAgentSessionsRoute LabChartsRoute: typeof LabChartsRoute @@ -1829,6 +1869,7 @@ interface LabRouteRouteChildren { } const LabRouteRouteChildren: LabRouteRouteChildren = { + LabAgentOverviewRoute: LabAgentOverviewRoute, LabAgentSessionRoute: LabAgentSessionRoute, LabAgentSessionsRoute: LabAgentSessionsRoute, LabChartsRoute: LabChartsRoute, @@ -1876,6 +1917,7 @@ const rootRouteChildren: RootRouteChildren = { SignInRoute: SignInRoute, SignUpRoute: SignUpRoute, AgentSessionsSessionIdRoute: AgentSessionsSessionIdRoute, + AgentSessionsOverviewRoute: AgentSessionsOverviewRoute, AlertsRuleIdRoute: AlertsRuleIdRoute, AlertsCreateRoute: AlertsCreateRoute, AnomaliesIncidentIdRoute: AnomaliesIncidentIdRoute, diff --git a/apps/web/src/routes/agent-sessions/index.tsx b/apps/web/src/routes/agent-sessions/index.tsx index deefb93f1..f3c379b2f 100644 --- a/apps/web/src/routes/agent-sessions/index.tsx +++ b/apps/web/src/routes/agent-sessions/index.tsx @@ -7,6 +7,7 @@ import { DashboardLayout } from "@/components/layout/dashboard-layout" import { AgentSessionsList } from "@/components/agent-sessions/agent-sessions-list" import { AgentSessionsFilterSidebar } from "@/components/agent-sessions/agent-sessions-filter-sidebar" import { AgentSessionsToolbar } from "@/components/agent-sessions/agent-sessions-toolbar" +import { AgentSessionsTabs } from "@/components/agent-sessions/tools/agent-sessions-tabs" import { agentSessionsFilterInputs, sortOptionFor, @@ -181,7 +182,14 @@ function AgentSessionsBody() { - {toolbar} + + {/* The Overview tab reads the whole population of these spans; this + page reads one session at a time. Two routes, one strip. */} +
+ + {toolbar} +
+ {Result.builder(firstPageResult) .onInitial(() => ( diff --git a/apps/web/src/routes/agent-sessions/overview.tsx b/apps/web/src/routes/agent-sessions/overview.tsx new file mode 100644 index 000000000..4bb7b174d --- /dev/null +++ b/apps/web/src/routes/agent-sessions/overview.tsx @@ -0,0 +1,335 @@ +import { useMemo, useState, type ReactNode } from "react" +import { createFileRoute, useNavigate } from "@tanstack/react-router" +import { Schema } from "effect" + +import { Skeleton } from "@maple/ui/components/ui/skeleton" +import { toEpochMs } from "@maple/ui/lib/time-format" + +import { + AgentOverviewView, + type AgentOverviewErrors, +} from "@/components/agent-sessions/overview/agent-overview-view" +import { OverviewMetricStripLoading } from "@/components/agent-sessions/overview/overview-metric-strip" +import { OverviewTrendsLoading } from "@/components/agent-sessions/overview/overview-trends" +import { DashboardLayout } from "@/components/layout/dashboard-layout" +import { NotFoundError } from "@/components/route-error" +import { + PageRefreshProvider, + usePageRefreshContext, +} from "@/components/time-range-picker/page-refresh-context" +import { + TimeRangeSearchFields, + applyTimeRangeSearch, + type TimeRangeSearch, +} from "@/components/time-range-picker/search" +import { sessionTimeRangeSearchMiddleware } from "@/components/time-range-picker/session-time-range" +import { TimeRangeHeaderControls } from "@/components/time-range-picker/time-range-header-controls" +import { useDetectedModels } from "@/hooks/use-detected-models" +import { useEffectiveTimeRange } from "@/hooks/use-effective-time-range" +import { useOrganizationFeatureFlags } from "@/hooks/use-organization-feature-flags" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { buildAgentOverviewData } from "@/lib/agent-sessions/overview-analytics" +import { + AGENT_OVERVIEW_DEFAULT_PRESET, + EMPTY_OVERVIEW_FACETS, + OverviewSearchFields, + compareEnabled, + overviewWindowLabel, + type AgentOverviewSearch, + type OverviewDimension, + type OverviewFacets, +} from "@/lib/agent-sessions/overview-search" +import { + useAgentOverview, + type OverviewTopSessionTab, +} from "@/lib/agent-sessions/use-agent-overview" +import { aiSessionsFacetsResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" + +const overviewSearchSchema = Schema.Struct({ + ...OverviewSearchFields, + ...TimeRangeSearchFields, +}) + +export const Route = createFileRoute("/agent-sessions/overview")({ + component: AgentOverviewPage, + validateSearch: Schema.toStandardSchemaV1(overviewSearchSchema), + search: { middlewares: [sessionTimeRangeSearchMiddleware()] }, +}) + +/** + * Behind the `agent_tracing` org rollout flag, gated exactly as the list and + * detail pages are: in the component rather than `beforeLoad` (router context + * carries no flags), `isLoaded` first so an entitled org gets no not-found + * flash, and no route `loader` — a loader would fire nine warehouse reads for + * orgs that are not entitled to the page at all. + */ +function AgentOverviewPage() { + const { flags, isLoaded } = useOrganizationFeatureFlags() + if (!isLoaded) return null + if (!flags.agentTracing) return + return +} + +function AgentOverviewPageContent() { + const search = Route.useSearch() + const navigate = useNavigate({ from: Route.fullPath }) + const preset = search.timePreset ?? AGENT_OVERVIEW_DEFAULT_PRESET + const { startTime, endTime } = useEffectiveTimeRange(search.startTime, search.endTime, preset) + // One object for the whole render tree below: it is a dependency of every + // selection memo down there, and a fresh literal defeats all of them. + const window = useMemo(() => ({ startTime, endTime }), [startTime, endTime]) + + const onSearchChange = (patch: Partial) => { + navigate({ search: (prev) => ({ ...prev, ...patch }) }) + } + + const handleTimeChange = ( + range: { startTime?: string; endTime?: string; presetValue?: string }, + options?: { replace?: boolean }, + ) => { + navigate({ + replace: options?.replace, + search: (prev) => ({ ...applyTimeRangeSearch(prev, range) }), + }) + } + + return ( + + + + + + + + } + /> + + + + + + ) +} + +/** A read as its section renders it: what it has, and the failure if it will + * never have anything. A read still in flight is neither — the section keeps + * its empty shape until it lands, and only a failed one is drawn as an error. */ +interface SectionRead { + readonly value: A + readonly error: unknown +} + +function sectionRead( + result: Result.Result, + map: (value: A) => B, + // `NoInfer` so the empty shape reads what the mapper returns rather than + // narrowing it: `[]` on its own infers `never[]`. + empty: NoInfer, +): SectionRead { + return Result.builder(result) + .onSuccess((value) => ({ value: map(value), error: undefined })) + .onError((error) => ({ value: empty, error })) + .orElse(() => ({ value: empty, error: undefined })) +} + +/** + * The nine reads, resolved. + * + * The **summary** is the one the page waits on: it is what the tiles, the chart + * headlines and the empty state are made of. Everything else degrades to empty + * rather than to a skeleton, so a slow breakdown leaves an empty table under a + * strip that is already drawn, not a page of grey boxes. + * + * A read that FAILED is not that, and is not an empty window either: each one + * is carried to its own section as an error, and only the summary's takes the + * body. Sections stay presentational — they receive a value and a failure, not + * a `Result`. + */ +function AgentOverviewBody({ + search, + window, + onSearchChange, + headerControls, +}: { + search: AgentOverviewSearch & TimeRangeSearch + window: { startTime: string; endTime: string } + onSearchChange: (patch: Partial) => void + headerControls: ReactNode +}) { + // The Top sessions tab lives here rather than in the table: it decides which + // list read runs, and only the open one should. + const [topSessionTab, setTopSessionTab] = useState("cost") + const results = useAgentOverview(search, window, topSessionTab) + // Every read on the page subscribes to this, so one retry re-runs all nine + // rather than only the one that failed. + const { reload } = usePageRefreshContext() + const windowMs = useMemo( + () => ({ startMs: toEpochMs(window.startTime), endMs: toEpochMs(window.endTime) }), + [window.startTime, window.endTime], + ) + const timeRange = useMemo( + () => ({ + startTime: search.startTime, + endTime: search.endTime, + timePreset: search.timePreset, + }), + [search.startTime, search.endTime, search.timePreset], + ) + // Read off the resolved window, not off the page's default preset: the + // resolver hands an absolute range straight back, so the default never + // named it and "prev 7d" beside a two-hour range would be a fiction. + const windowLabel = overviewWindowLabel(timeRange, windowMs.endMs - windowMs.startMs) + + // The selects' options come from the sessions facets — the same counted + // lists the list page's sidebar uses, unfiltered so picking one model does + // not erase the others. Plain `useAtomValue` keeps them off the Reload + // subscription, so a manual refresh cannot rebuild a select under a click. + const facetsResult = useAtomValue(aiSessionsFacetsResultAtom({ data: window })) + const facetsRead: SectionRead = sectionRead( + facetsResult, + (value) => ({ + model: value.models, + agent: value.agents, + service: value.services, + framework: value.vendors, + environment: value.environments, + tool: value.tools, + }), + EMPTY_OVERVIEW_FACETS, + ) + + const breakdownReads = useMemo( + () => + results.breakdowns.map((breakdown) => ({ + dimension: breakdown.dimension, + read: sectionRead( + breakdown.result, + (value) => ({ entries: value.entries, totalKeys: value.totalKeys }), + { entries: [], totalKeys: 0 }, + ), + })), + [results.breakdowns], + ) + const breakdowns = useMemo( + () => breakdownReads.map(({ dimension, read }) => ({ dimension, ...read.value })), + [breakdownReads], + ) + const breakdownErrors = useMemo(() => { + const errors: Partial> = {} + for (const { dimension, read } of breakdownReads) { + if (read.error !== undefined) errors[dimension] = read.error + } + return errors + }, [breakdownReads]) + // Held across renders like the breakdowns above: the board's view model is + // memoised on these, and a fresh empty array every render rebuilds it. + const modelMixRead = useMemo( + () => sectionRead(results.modelMix, (value) => value.rows, []), + [results.modelMix], + ) + const modelMix = modelMixRead.value + const sessionsRead = useMemo( + () => sectionRead(results.topSessions, (value) => value.data, []), + [results.topSessions], + ) + const sessions = sessionsRead.value + // One detection read for the whole table, exactly as the Sessions list does + // it — the models a row ran are resolved to their vendor and display name. + const sessionModels = useMemo(() => sessions.flatMap((session) => session.models), [sessions]) + const detectModel = useDetectedModels(sessionModels) + + // Built once per resolved read rather than once per render: the view model + // carries `series`, `previousSeries` and `modelMix`, and the trends grid + // memoises nine plot specs and a shared axis on their identity. + const resolved = Result.isSuccess(results.summary) ? results.summary : undefined + const summary = resolved?.value + const compare = compareEnabled(search) + const data = useMemo( + () => + summary === undefined + ? undefined + : buildAgentOverviewData({ + current: summary.current, + previous: summary.previous, + series: summary.series, + previousSeries: summary.previousSeries, + modelMix, + breakdowns, + // The width the buckets were actually cut at, echoed back rather + // than re-derived for the axis. + bucketSeconds: summary.bucketSeconds, + windowMs, + windowLabel, + compare, + }), + [summary, modelMix, breakdowns, windowMs, windowLabel, compare], + ) + + const summaryError = Result.builder(results.summary) + .onError((error) => error) + .orElse(() => undefined) + + // Shaped like what lands, so nothing reflows when it does: the strip keeps + // its seven tiles and the grid its nine cells. Only while the summary is + // still in flight — a failed one keeps the page's chrome instead. + if (data === undefined && summaryError === undefined) { + return ( +
+
+ + +
+ + +
+ + {Array.from({ length: 5 }).map((_, index) => ( + + ))} +
+
+ ) + } + + const errors: AgentOverviewErrors = { + summary: summaryError, + facets: facetsRead.error !== undefined, + breakdowns: breakdownErrors, + modelMix: modelMixRead.error, + topSessions: sessionsRead.error, + } + + return ( + + ) +} diff --git a/apps/web/src/routes/lab/agent-overview.tsx b/apps/web/src/routes/lab/agent-overview.tsx new file mode 100644 index 000000000..894cabaec --- /dev/null +++ b/apps/web/src/routes/lab/agent-overview.tsx @@ -0,0 +1,5 @@ +import { createFileRoute } from "@tanstack/react-router" + +import { AgentOverviewLab } from "@/lab/agent-overview-lab" + +export const Route = createFileRoute("/lab/agent-overview")({ component: AgentOverviewLab }) diff --git a/packages/domain/src/http/ai-sessions.ts b/packages/domain/src/http/ai-sessions.ts index a44b45134..de689bb3a 100644 --- a/packages/domain/src/http/ai-sessions.ts +++ b/packages/domain/src/http/ai-sessions.ts @@ -2,6 +2,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi" import { Schema } from "effect" import { AiAgentSpanSchema, AiGenAiValuesSchema } from "../gen-ai" import { TinybirdDateTime } from "../query-engine" +import { BucketSeconds } from "./query-engine" import { SessionAuthorization } from "./current-tenant" import { HttpTaggedError } from "./error-policy" import { warehouseReadHttpErrors } from "./warehouse" @@ -530,6 +531,330 @@ export class AiSessionTooLargeError extends HttpTaggedError + +export const AiOverviewSeriesPoint = Schema.Struct({ + /** ISO-8601 with a literal `Z`, the shape every Maple timeseries emits. */ + bucket: Schema.String, + ...aiOverviewMeasures, +}) +export type AiOverviewSeriesPoint = Schema.Schema.Type + +/** The selection every bucketed overview read takes: the window, the width its + * buckets are cut at, and the filters. */ +const aiOverviewBucketedSelection = { + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + /** Whole seconds, greater than zero — it reaches `toStartOfInterval` as an + * `INTERVAL n SECOND` literal, so a fraction is a 400 and not a 500. */ + bucketSeconds: BucketSeconds, + ...aiOverviewSelection, +} + +/** + * Whether a pair of bounds is a window an overview read can run, or why not. + * + * Three rules, and the first is the reason this exists at all: + * + * - Both bounds have to be datetimes the CALENDAR admits, not just the pattern. + * `TinybirdDateTime` takes `2026-13-45 99:99:99`, which parses to NaN — and + * every overview read derives its comparison window from these two, where a + * NaN is a `new Date(NaN).toISOString()` and a 500. + * - An inverted window reaches the partition predicate as given and answers + * empty, which reads as "nothing ran" rather than as the bad request it is. + * - Past `ai_trace_index`'s retention the read is a scan of partitions the + * index cannot hold rows for — and the comparison window doubles it. + * + * Exported because a client mirroring these bounds has to run the same rule: a + * window this refuses makes the request constructor throw, where a mirror that + * checks first turns it into a failure the page can render. + */ +export const isAiOverviewWindow = (startTime: string, endTime: string): true | string => { + const extentMs = tinybirdDateTimeMs(endTime) - tinybirdDateTimeMs(startTime) + if (Number.isNaN(extentMs)) return "startTime and endTime must be valid datetimes" + if (extentMs < 0) return "startTime must not be after endTime" + // No extent cap: every overview read is an index range read, and the index's + // retention already bounds how much a wide window can cost. + return true +} + +const aiOverviewWindowValid = Schema.makeFilter( + (request: { readonly startTime: string; readonly endTime: string }) => + isAiOverviewWindow(request.startTime, request.endTime), + { identifier: "OverviewWindowValid" }, +) + +export class AiOverviewSummaryRequest extends Schema.Class( + "AiOverviewSummaryRequest", +)(Schema.Struct(aiOverviewBucketedSelection).check(aiOverviewWindowValid)) {} + +export class AiOverviewSummaryResponse extends Schema.Class( + "AiOverviewSummaryResponse", +)({ + /** Echoed back, so a client rendering an axis reads the width the buckets + * were actually cut at rather than re-deriving it. */ + bucketSeconds: BucketSeconds, + /** The whole selected window. */ + current: AiOverviewMeasures, + /** + * The window of equal length immediately before the caller's, measured by + * the same read — the deltas the tiles show. Zeros where nothing ran then, + * which the client renders as "no comparison" rather than a -100%. + */ + previous: AiOverviewMeasures, + /** + * One point per bucket that had a session, oldest first. A session — and + * its netted usage, its calls and its failures — belongs to the bucket its + * FIRST span started in, so the points sum to `current` rather than + * counting a long session in every bucket it touched. Quantiles are the + * exception and cannot be summed at all, which is why `current` comes from + * its own un-bucketed read and not from these. + */ + series: Schema.Array(AiOverviewSeriesPoint), + /** The same, over the previous window and at the same bucket width. */ + previousSeries: Schema.Array(AiOverviewSeriesPoint), +}) {} + +/** Which dimension the breakdown groups by. Each is a column of + * `ai_trace_index`; there is no provider column — a model maps to its + * provider client-side. */ +export const AiOverviewDimension = Schema.Literals([ + "model", + "agent", + "service", + "environment", + "vendor", + "tool", +]) +export type AiOverviewDimension = Schema.Schema.Type + +/** Rows one breakdown returns, and the default. The page shows a table, not a + * catalogue: `totalKeys` is what lets it say "+ N more" off the same read. */ +export const AI_OVERVIEW_BREAKDOWN_MAX = 12 + +/** + * One key of a breakdown, measured over both windows. + * + * A row carries the whole measure set, but which of them MEAN anything depends + * on the dimension, because `model` and `tool` restrict the population to the + * spans that can carry the key — model calls and tool calls respectively: + * + * - `model`: `toolCalls` and `erroredToolCalls` are structurally 0 (a model + * call is not a tool call), and the usage and call measures are the row's + * subject. + * - `tool`: `llmCalls`, `llmCallSpans` and `erroredLlmCalls` are structurally + * 0 (a tool call is not a model call), and `cost`, `tokens` and + * `pricedLlmCalls` are 0 for every tool span that reports no usage, which is + * all of them in practice. `toolCalls` and `erroredToolCalls` are the row's + * subject. + * - `agent`, `service`, `environment`, `vendor`: every agent span carries the + * key, so every measure is meaningful. + * + * `sessions`, `erroredSessions` and `sessionDurationP*Ns` are always over THIS + * key's spans: a session appears under every key it used, its failures are the + * ones its spans under this key carried, and its extent runs from the first of + * those spans to the last rather than across the whole session. A client + * renders the columns the dimension supports rather than a column of zeros. + */ +export const AiOverviewBreakdownRow = Schema.Struct({ + /** + * The dimension's value. `''` is a real key, not a gap — a span that + * carries no value for this dimension — and the page renders it as + * unattributed rather than hiding it. + */ + key: Schema.String, + current: AiOverviewMeasures, + /** The same key over the previous window; zeros where it did not appear. */ + previous: AiOverviewMeasures, +}) +export type AiOverviewBreakdownRow = Schema.Schema.Type + +export class AiOverviewBreakdownRequest extends Schema.Class( + "AiOverviewBreakdownRequest", +)( + Schema.Struct({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + dimension: AiOverviewDimension, + limit: Schema.optionalKey( + Schema.Number.check( + Schema.isInt(), + Schema.isBetween({ minimum: 1, maximum: AI_OVERVIEW_BREAKDOWN_MAX }), + ), + ), + ...aiOverviewSelection, + }).check(aiOverviewWindowValid), +) {} + +export class AiOverviewBreakdownResponse extends Schema.Class( + "AiOverviewBreakdownResponse", +)({ + dimension: AiOverviewDimension, + /** + * The busiest keys by session count, most sessions first. + * + * Rows OVERLAP and need not sum to the totals: a session that used two + * models is a session under each of them. What does not overlap is the + * usage — a model call's tokens are netted under the model that reported + * them, so the cost column splits rather than repeats. + */ + rows: Schema.Array(AiOverviewBreakdownRow), + /** Distinct keys in the current window, so the table can say how many it + * is not showing. */ + totalKeys: OverviewCount, +}) {} + +/** One bucket's share of one model. */ +export const AiOverviewModelMixPoint = Schema.Struct({ + /** ISO-8601 with a literal `Z`, the shape every Maple timeseries emits. */ + bucket: Schema.String, + /** + * The model the spans named, or `other` — the band the read folds every + * model past the busiest few into, so a bucket answers a bounded number of + * rows. Never `''`: a call that named no model has no share of a model mix, + * and the read leaves it out. + */ + model: Schema.String, + /** Model-call SPANS, counted raw. See {@link AiOverviewModelMixResponse}. */ + llmCallSpans: OverviewCount, +}) +export type AiOverviewModelMixPoint = Schema.Schema.Type + +export class AiOverviewModelMixRequest extends Schema.Class( + "AiOverviewModelMixRequest", +)(Schema.Struct(aiOverviewBucketedSelection).check(aiOverviewWindowValid)) {} + +export class AiOverviewModelMixResponse extends Schema.Class( + "AiOverviewModelMixResponse", +)({ + /** Echoed back, so a client rendering an axis reads the width the buckets + * were actually cut at rather than re-deriving it. */ + bucketSeconds: BucketSeconds, + /** + * One row per (bucket, model) the window saw, oldest bucket first and the + * busiest model of a bucket first. + * + * The share of model SPANS — the raw population the summary reports as + * `llmCallSpans`, so a gateway's mirror of a call is counted under the model + * it names, twice. Netting would charge a mirrored call to one model alone + * and leave the bands disagreeing with the error rate above them. A band's + * share is its count over its bucket's, and the client folds the minor + * models into an "other" band rather than plotting a line per model. + */ + rows: Schema.Array(AiOverviewModelMixPoint), +}) {} + export class AiSessionsInternalApiGroup extends HttpApiGroup.make("aiSessionsInternal") .add( HttpApiEndpoint.post("list", "/list", { @@ -566,5 +891,26 @@ export class AiSessionsInternalApiGroup extends HttpApiGroup.make("aiSessionsInt error: warehouseReadHttpErrors, }), ) + .add( + HttpApiEndpoint.post("overviewSummary", "/overview/summary", { + payload: AiOverviewSummaryRequest, + success: AiOverviewSummaryResponse, + error: warehouseReadHttpErrors, + }), + ) + .add( + HttpApiEndpoint.post("overviewBreakdown", "/overview/breakdown", { + payload: AiOverviewBreakdownRequest, + success: AiOverviewBreakdownResponse, + error: warehouseReadHttpErrors, + }), + ) + .add( + HttpApiEndpoint.post("overviewModelMix", "/overview/model-mix", { + payload: AiOverviewModelMixRequest, + success: AiOverviewModelMixResponse, + error: warehouseReadHttpErrors, + }), + ) .prefix("/internal/ai-sessions") .middleware(SessionAuthorization) {} diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index a76088017..93a07bb81 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -36,7 +36,7 @@ import { FunnelBreakdownBy, FunnelKeyBy, FunnelStep } from "@maple/query-model" * of a 400. `packages/domain/src/query-engine.ts` already had this right; these * declarations did not. */ -const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)).pipe( +export const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)).pipe( Schema.annotate({ identifier: "BucketSeconds", description: "Timeseries bucket width in whole seconds, greater than zero.", diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index f39c1bbd9..851eca785 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -1,3 +1,1208 @@ +-- builder:ai-overview:aiOverviewBreakdownQuery:model +SELECT + 'current' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.Model) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_current + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.Model) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'previous' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.Model) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_previous + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.Model) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'keys' AS period, + '' AS key, + count() AS keyCount, + 0 AS sessions, + 0 AS erroredSessions, + 0 AS llmCalls, + 0 AS llmCallSpans, + 0 AS erroredLlmCalls, + 0 AS toolCalls, + 0 AS erroredToolCalls, + 0 AS cost, + 0 AS pricedLlmCalls, + 0 AS tokens, + 0 AS inputTokens, + 0 AS cacheReadTokens, + 0 AS cacheWriteTokens, + 0 AS outputTokens, + 0 AS reasoningTokens, + 0 AS sessionDurationP50Ns, + 0 AS sessionDurationP95Ns + FROM (SELECT + toString(ai_trace_index.Model) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + GROUP BY rankKey) AS window_keys +FORMAT JSON + +-- builder:ai-overview:aiOverviewBreakdownQuery:service +SELECT + 'current' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ServiceName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId, key) AS session_rows) AS netted_current + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ServiceName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'previous' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ServiceName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' + GROUP BY sessionId, key) AS session_rows) AS netted_previous + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ServiceName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 12) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'keys' AS period, + '' AS key, + count() AS keyCount, + 0 AS sessions, + 0 AS erroredSessions, + 0 AS llmCalls, + 0 AS llmCallSpans, + 0 AS erroredLlmCalls, + 0 AS toolCalls, + 0 AS erroredToolCalls, + 0 AS cost, + 0 AS pricedLlmCalls, + 0 AS tokens, + 0 AS inputTokens, + 0 AS cacheReadTokens, + 0 AS cacheWriteTokens, + 0 AS outputTokens, + 0 AS reasoningTokens, + 0 AS sessionDurationP50Ns, + 0 AS sessionDurationP95Ns + FROM (SELECT + toString(ai_trace_index.ServiceName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY rankKey) AS window_keys +FORMAT JSON + +-- builder:ai-overview:aiOverviewBreakdownQuery:tool +SELECT + 'current' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ToolName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_current + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ToolName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 5) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'previous' AS period, + key AS key, + 0 AS keyCount, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + toString(ai_trace_index.ToolName) AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY sessionId, key) AS session_rows) AS netted_previous + WHERE key IN (SELECT + rankKey AS topKey + FROM (SELECT + toString(ai_trace_index.ToolName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY rankKey + ORDER BY rankSessions DESC, rankKey ASC + LIMIT 5) AS top_keys) + GROUP BY key +UNION ALL +SELECT + 'keys' AS period, + '' AS key, + count() AS keyCount, + 0 AS sessions, + 0 AS erroredSessions, + 0 AS llmCalls, + 0 AS llmCallSpans, + 0 AS erroredLlmCalls, + 0 AS toolCalls, + 0 AS erroredToolCalls, + 0 AS cost, + 0 AS pricedLlmCalls, + 0 AS tokens, + 0 AS inputTokens, + 0 AS cacheReadTokens, + 0 AS cacheWriteTokens, + 0 AS outputTokens, + 0 AS reasoningTokens, + 0 AS sessionDurationP50Ns, + 0 AS sessionDurationP95Ns + FROM (SELECT + toString(ai_trace_index.ToolName) AS rankKey, + uniqExact(if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)) AS rankSessions + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsToolCall = 1 + GROUP BY rankKey) AS window_keys +FORMAT JSON + +-- builder:ai-overview:aiOverviewModelMixQuery:default +SELECT + formatDateTime(toStartOfInterval(ai_trace_index.Timestamp, INTERVAL 300 SECOND), '%Y-%m-%dT%H:%i:%S.%fZ') AS bucket, + if(toString(ai_trace_index.Model) IN (SELECT + rankModel AS topModel + FROM (SELECT + toString(ai_trace_index.Model) AS rankModel, + count() AS rankSpans + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + AND ai_trace_index.Model != '' + GROUP BY rankModel + ORDER BY rankSpans DESC, rankModel ASC + LIMIT 5) AS top_models), toString(ai_trace_index.Model), 'other') AS model, + count() AS llmCallSpans + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + AND ai_trace_index.IsLlmCall = 1 + AND ai_trace_index.Model != '' + GROUP BY bucket, model + ORDER BY bucket ASC, llmCallSpans DESC + LIMIT 4000 + FORMAT JSON + +-- builder:ai-overview:aiOverviewSeriesQuery:default +SELECT * FROM ( +SELECT + 'current' AS period, + formatDateTime(toStartOfInterval(sessionStart, INTERVAL 300 SECOND), '%Y-%m-%dT%H:%i:%S.%fZ') AS bucket, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId) AS session_rows) AS netted_current + GROUP BY bucket +UNION ALL +SELECT + 'previous' AS period, + formatDateTime(toStartOfInterval(sessionStart, INTERVAL 300 SECOND), '%Y-%m-%dT%H:%i:%S.%fZ') AS bucket, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' + GROUP BY sessionId) AS session_rows) AS netted_previous + GROUP BY bucket +) +ORDER BY period ASC, bucket ASC +FORMAT JSON + +-- builder:ai-overview:aiOverviewTotalsQuery:default +SELECT + 'current' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId) AS session_rows) AS netted_current +UNION ALL +SELECT + 'previous' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId) AS selected_traces) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' + GROUP BY sessionId) AS session_rows) AS netted_previous +FORMAT JSON + +-- builder:ai-overview:aiOverviewTotalsQuery:every-filter +SELECT + 'current' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2026-01-01 10:30:00' + AND ai_trace_index.Timestamp <= '2026-01-03 14:15:00' + GROUP BY sessionId) AS session_rows) AS netted_current +UNION ALL +SELECT + 'previous' AS period, + count() AS sessions, + countIf(errorSpans > 0) AS erroredSessions, + sum(toFloat64(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 2)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2)), arrayFilter(n -> n.1 != '', netted))))))) AS llmCalls, + sum(llmCallSpans) AS llmCallSpans, + sum(erroredLlmCalls) AS erroredLlmCalls, + sum(toolCalls) AS toolCalls, + sum(erroredToolCalls) AS erroredToolCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 4)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.4), arrayFilter(n -> n.1 != '', netted)))))) AS cost, + sum(arraySum(arrayMap(n -> toFloat64(n.2 AND n.4 > 0), arrayFilter(n -> n.1 = '', netted))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, toFloat64(n.2 AND n.4 > 0)), arrayFilter(n -> n.1 != '', netted)))))) AS pricedLlmCalls, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 3)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.3), arrayFilter(n -> n.1 != '', netted)))))) AS tokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 5)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.5), arrayFilter(n -> n.1 != '', netted)))))) AS inputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 6)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.6), arrayFilter(n -> n.1 != '', netted)))))) AS cacheReadTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 7)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.7), arrayFilter(n -> n.1 != '', netted)))))) AS cacheWriteTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 8)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.8), arrayFilter(n -> n.1 != '', netted)))))) AS outputTokens, + sum(arraySum(tupleElement(arrayFilter(n -> n.1 = '', netted), 9)) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, n.9), arrayFilter(n -> n.1 != '', netted)))))) AS reasoningTokens, + ifNull(ifNotFinite(quantile(0.5)(sessionDurationNs), 0), 0) AS sessionDurationP50Ns, + ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0) AS sessionDurationP95Ns + FROM (SELECT + key AS key, + sessionStart AS sessionStart, + sessionDurationNs AS sessionDurationNs, + errorSpans AS errorSpans, + toolCalls AS toolCalls, + erroredToolCalls AS erroredToolCalls, + erroredLlmCalls AS erroredLlmCalls, + llmCallSpans AS llmCallSpans, + arrayMap(r -> tuple(r.5, r.6 = 1 AND if((r.3 > 0 OR r.4 > 0), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))) > 0 OR greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))) > 0, NOT has(reportingIds, r.2)), greatest(0., r.3 - arrayElement(tupleElement(childClaims, 2), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.4 - arrayElement(tupleElement(childClaims, 3), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.7 - arrayElement(tupleElement(childClaims, 4), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.8 - arrayElement(tupleElement(childClaims, 5), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.9 - arrayElement(tupleElement(childClaims, 6), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.10 - arrayElement(tupleElement(childClaims, 7), indexOf(tupleElement(childClaims, 1), r.1))), greatest(0., r.11 - arrayElement(tupleElement(childClaims, 8), indexOf(tupleElement(childClaims, 1), r.1)))), reporters) AS netted + FROM (SELECT + if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId) AS sessionId, + '' AS key, + min(ai_trace_index.Timestamp) AS sessionStart, + max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs, + sum(ai_trace_index.IsError) AS errorSpans, + sum(ai_trace_index.IsToolCall) AS toolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsToolCall = 1) AS erroredToolCalls, + sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls, + sum(ai_trace_index.IsLlmCall) AS llmCallSpans, + groupArrayIf(2000)(tuple(ai_trace_index.SpanId, ai_trace_index.ParentSpanId, ai_trace_index.Tokens, ai_trace_index.Cost, ai_trace_index.ResponseId, ai_trace_index.IsLlmCall, ai_trace_index.InputTokens, ai_trace_index.CacheReadTokens, ai_trace_index.CacheWriteTokens, ai_trace_index.OutputTokens, ai_trace_index.ReasoningTokens), ((ai_trace_index.Tokens > 0 OR ai_trace_index.Cost > 0) OR ai_trace_index.IsLlmCall = 1)) AS reporters, + arrayReduce('sumMap', arrayMap(c -> [c.2], reporters), arrayMap(c -> [c.3], reporters), arrayMap(c -> [c.4], reporters), arrayMap(c -> [c.7], reporters), arrayMap(c -> [c.8], reporters), arrayMap(c -> [c.9], reporters), arrayMap(c -> [c.10], reporters), arrayMap(c -> [c.11], reporters)) AS childClaims, + tupleElement(arrayFilter(p -> p.3 > 0 OR p.4 > 0, reporters), 1) AS reportingIds + FROM ai_trace_index + INNER JOIN (SELECT + TraceId AS TraceId, + rawSessionId AS rawSessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS selected_traces + WHERE if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) IN (SELECT + if(rawSessionId = '', concat('trace:', TraceId), rawSessionId) AS sessionId + FROM (SELECT + TraceId AS TraceId, + max(SessionId) AS rawSessionId, + sum(IsError) AS errorSpans + FROM ai_trace_index + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2025-12-30 06:45:00' + AND Timestamp < '2026-01-01 10:30:00' + GROUP BY TraceId + HAVING countIf(VendorId IN ('eve')) > 0 + AND countIf(ServiceName IN ('maple-slack-agent')) > 0 + AND countIf(DeploymentEnv IN ('production')) > 0 + AND countIf(Model IN ('gpt-5.5')) > 0 + AND countIf(AgentName IN ('billing-agent')) > 0 + AND countIf(ToolName IN ('send_email')) > 0) AS errored_traces + GROUP BY sessionId + HAVING sum(errorSpans) > 0)) AS trace ON ai_trace_index.TraceId = trace.TraceId + WHERE ai_trace_index.OrgId = 'org_sql_catalog' + AND ai_trace_index.Timestamp >= '2025-12-30 06:45:00' + AND ai_trace_index.Timestamp < '2026-01-01 10:30:00' + GROUP BY sessionId) AS session_rows) AS netted_previous +FORMAT JSON + -- builder:ai-sessions:aiSessionDetailsQuery:default SELECT if(index_traces.rawSessionId = '', concat('trace:', session_traces.traceId), index_traces.rawSessionId) AS sessionId, diff --git a/packages/query-engine-integrations/src/ai/ai-overview.test.ts b/packages/query-engine-integrations/src/ai/ai-overview.test.ts new file mode 100644 index 000000000..e630e60f1 --- /dev/null +++ b/packages/query-engine-integrations/src/ai/ai-overview.test.ts @@ -0,0 +1,350 @@ +import { describe, expect, it } from "vitest" +import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-clickhouse" +import { AI_OVERVIEW_BREAKDOWN_MAX } from "@maple/domain/http" +import { + aiOverviewBreakdownQuery, + aiOverviewModelMixQuery, + aiOverviewSeriesQuery, + aiOverviewTotalsQuery, + AI_OVERVIEW_MODEL_MIX_MAX_ROWS, +} from "./ai-overview" + +const params = { + orgId: "org_1", + startTime: "2026-08-18 00:00:00", + endTime: "2026-08-19 23:59:59", + prevStartTime: "2026-08-16 00:00:01", + prevEndTime: "2026-08-18 00:00:00", +} + +/** The series is the only read that cuts buckets. */ +const seriesParams = { ...params, bucketSeconds: 300 } + +/** The sessions list's key, resolved per trace — the same expression + * `aiSessionPageQuery` groups on, so a number here reconciles with a row + * there. */ +const SESSION_KEY = + "if(trace.rawSessionId = '', concat('trace:', ai_trace_index.TraceId), trace.rawSessionId)" + +/** The same key, resolved off the trace rollup's own columns — where the + * `hasErrors` test is applied. */ +const ROLLUP_SESSION_KEY = "if(rawSessionId = '', concat('trace:', TraceId), rawSessionId)" + +/** `OrgId = 'x'` on every level that reads a table — a subquery contributes + * nothing to the outer query's scope. */ +const orgPredicateCount = (sql: string) => sql.split("OrgId = 'org_1'").length - 1 + +const totalsSql = (opts = {}) => compileUnionUnsafe(aiOverviewTotalsQuery(opts), params).sql +const seriesSql = (opts = {}) => compileUnionUnsafe(aiOverviewSeriesQuery(opts), seriesParams).sql + +describe("overview population", () => { + it("reads ai_trace_index alone", () => { + for (const sql of [totalsSql(), seriesSql()]) { + expect(sql).toContain("FROM ai_trace_index") + // The moment an overview read reaches the span tables it costs what the + // sessions fan-out costs — seconds, per partition, over a month. + expect(sql).not.toContain("trace_detail_spans") + expect(sql).not.toContain("SpanAttributes") + expect(sql).not.toContain("__PARAM_") + } + }) + + it("resolves the session per trace, with the sessions list's own key", () => { + const sql = totalsSql() + + // `max(SessionId)` per trace, because the id sits on the turn-owning span + // and every other row of the trace reads ''. + expect(sql).toContain("max(SessionId) AS rawSessionId") + expect(sql).toContain(`${SESSION_KEY} AS sessionId`) + }) + + it("nets usage and model calls the way the sessions list nets them", () => { + const sql = totalsSql() + + // The reporters, the two lookups taken off them once per session, and the + // netting — a naive sum(Cost) double-counts every wrapper roll-up. + expect(sql).toContain("AS reporters") + expect(sql).toContain("AS childClaims") + expect(sql).toContain("AS reportingIds") + expect(sql).toContain("AS netted") + expect(sql).toContain("maxMap") + expect(sql).not.toContain("sum(Cost)") + expect(sql).not.toContain("sum(Tokens)") + }) + + it("scopes every level that reads the table to the org", () => { + // Two levels per branch — the trace rollup and the session rows — and two + // branches. + expect(orgPredicateCount(totalsSql())).toBe(4) + expect(compileUnionUnsafe(aiOverviewTotalsQuery(), params).tenantScope).toBe("single-tenant") + expect(compileUnionUnsafe(aiOverviewSeriesQuery(), seriesParams).tenantScope).toBe("single-tenant") + expect(compileUnionUnsafe(aiOverviewBreakdownQuery({ dimension: "model" }), params).tenantScope).toBe( + "single-tenant", + ) + }) + + it("applies each filter as a per-trace existence test, and none when none is given", () => { + const sql = totalsSql({ + vendorIds: ["eve"], + serviceNames: ["agent-runner"], + deploymentEnvs: ["production"], + models: ["gpt-5.5"], + agentNames: ["billing-agent"], + toolNames: ["send_email"], + }) + + // HAVING, not WHERE: a row predicate would narrow the rows the session id + // is read from, and a model ANDed with a tool can never match one row. + expect(sql).toContain("countIf(VendorId IN ('eve')) > 0") + expect(sql).toContain("countIf(ServiceName IN ('agent-runner')) > 0") + expect(sql).toContain("countIf(DeploymentEnv IN ('production')) > 0") + expect(sql).toContain("countIf(Model IN ('gpt-5.5')) > 0") + expect(sql).toContain("countIf(AgentName IN ('billing-agent')) > 0") + expect(sql).toContain("countIf(ToolName IN ('send_email')) > 0") + + const unfiltered = totalsSql() + expect(unfiltered).not.toContain("HAVING") + expect(unfiltered).not.toContain("countIf(VendorId") + }) + + it("bounds the comparison window half-open, so the boundary is measured once", () => { + const sql = totalsSql() + + // `[prevStartTime, startTime)`: the previous window ends where the + // caller's begins, and every level of the current branch still takes the + // closed window every other Maple read takes. + expect(sql).toContain(`Timestamp >= '${params.prevStartTime}'`) + expect(sql).toContain(`Timestamp < '${params.prevEndTime}'`) + expect(sql).toContain(`Timestamp <= '${params.endTime}'`) + // `prevEndTime` IS `startTime`, so a row on it would otherwise land in + // both windows. + expect(sql).not.toContain(`Timestamp <= '${params.prevEndTime}'`) + }) + + it("selects the sessions that failed with the list's session-level rule", () => { + const sql = totalsSql({ hasErrors: true }) + + // A session-level test, not a trace-level one: a session spans traces and + // the list matches it when any of its agent spans failed — summed over the + // trace rollup, which is the same sum one level up. + expect(sql).toContain("HAVING sum(errorSpans) > 0") + expect(sql).toContain(`${ROLLUP_SESSION_KEY} IN (SELECT`) + expect(totalsSql()).not.toContain("HAVING sum(errorSpans) > 0") + }) + + it("applies the failed-session test once, at the trace level", () => { + const withErrors = totalsSql({ hasErrors: true }) + + // The errored set is a read of its own; deriving it at each level that + // filters is the same scan four times over. Every level above joins the + // already-filtered trace set instead, so the set is built once per window. + expect(withErrors.split("HAVING sum(errorSpans) > 0").length - 1).toBe(2) + expect(orgPredicateCount(withErrors)).toBe(6) + const breakdown = compileUnionUnsafe( + aiOverviewBreakdownQuery({ dimension: "service", hasErrors: true }), + params, + ).sql + expect(breakdown.split("FROM ai_trace_index").length - 1).toBe(15) + }) +}) + +describe("the measures every grouping reports", () => { + it("counts a session once and files it under the bucket it started in", () => { + const sql = seriesSql() + + // `count()`, not `uniqExact`: the level below is already one row per + // session, and a session has exactly one first span — so the buckets sum + // to the totals. + expect(sql).toContain("count() AS sessions") + expect(sql).toContain("countIf(errorSpans > 0) AS erroredSessions") + expect(sql).toContain("min(ai_trace_index.Timestamp) AS sessionStart") + expect(sql).toContain("toStartOfInterval(sessionStart, INTERVAL 300 SECOND)") + expect(sql).toContain("GROUP BY bucket") + // The totals are their own un-bucketed read, because quantiles do not + // merge — a p95 folded from the series is not a p95. + expect(totalsSql()).not.toContain("toStartOfInterval") + }) + + it("gives the model-call failures a denominator of their own population", () => { + const sql = totalsSql() + + // The numerator is a span `sumIf` — a failure cannot be netted, the index + // carries no error flag into the reporters — so the denominator counts + // the same spans. Against the netted `llmCalls`, a mirrored call that + // failed on both observations is a rate above 100%. + expect(sql).toContain( + "sumIf(ai_trace_index.IsError, ai_trace_index.IsLlmCall = 1) AS erroredLlmCalls", + ) + expect(sql).toContain("sum(ai_trace_index.IsLlmCall) AS llmCallSpans") + expect(sql).toContain("sum(llmCallSpans) AS llmCallSpans") + }) + + it("guards every quantile against the empty group", () => { + const sql = totalsSql() + + // A quantile over no rows is NULL, which the row schema refuses. + for (const measure of ["sessionDurationP50Ns", "sessionDurationP95Ns"]) { + expect(sql).toContain(`AS ${measure}`) + } + expect(sql).toContain("ifNull(ifNotFinite(quantile(0.95)(sessionDurationNs), 0), 0)") + // The session's extent is the only quantile the page reads: nothing + // renders a per-call latency, and collecting one was an array of every + // model call of every session. + expect(sql).not.toContain("quantileArray") + expect(sql).not.toContain("llmDurations") + }) + + it("measures the extent of the session, not the start of its last span", () => { + // Without the `+ Duration` a session whose trace is one long span reports + // a duration of 0, and every other session under-reports by the + // last-starting span's own duration. + expect(totalsSql()).toContain( + "max(toUnixTimestamp64Nano(ai_trace_index.Timestamp) + toInt64(ai_trace_index.Duration)) - toUnixTimestamp64Nano(min(ai_trace_index.Timestamp)) AS sessionDurationNs", + ) + }) +}) + +describe("the breakdown's dimensions", () => { + const breakdownSql = (opts: Parameters[0]) => + compileUnionUnsafe(aiOverviewBreakdownQuery(opts), params).sql + + it("keys each dimension by the column the span carries it in", () => { + expect(breakdownSql({ dimension: "model" })).toContain("toString(ai_trace_index.Model) AS key") + expect(breakdownSql({ dimension: "agent" })).toContain("toString(ai_trace_index.AgentName) AS key") + expect(breakdownSql({ dimension: "service" })).toContain( + "toString(ai_trace_index.ServiceName) AS key", + ) + expect(breakdownSql({ dimension: "environment" })).toContain( + "toString(ai_trace_index.DeploymentEnv) AS key", + ) + expect(breakdownSql({ dimension: "vendor" })).toContain("toString(ai_trace_index.VendorId) AS key") + expect(breakdownSql({ dimension: "tool" })).toContain("toString(ai_trace_index.ToolName) AS key") + }) + + it("reads a model over model calls and a tool over tool calls, and the rest over every span", () => { + // The predicate, not the `sumIf` measures every read carries: a model + // keys off a column only a model call fills, and a tool off one only a + // tool call fills, so the population is narrowed rather than left to + // answer `''` for every other span. + expect(breakdownSql({ dimension: "model" })).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(breakdownSql({ dimension: "tool" })).toContain("AND ai_trace_index.IsToolCall = 1") + const byService = breakdownSql({ dimension: "service" }) + expect(byService).not.toContain("AND ai_trace_index.IsLlmCall = 1") + expect(byService).not.toContain("AND ai_trace_index.IsToolCall = 1") + }) + + it("measures both windows over the keys the current window ranked, and counts the rest", () => { + const sql = breakdownSql({ dimension: "model" }) + + // The previous branch is restricted to the same keys, so a key that + // stopped being used still shows what it cost. + expect(sql.split("key IN (SELECT").length - 1).toBe(2) + expect(sql).toContain(`LIMIT ${AI_OVERVIEW_BREAKDOWN_MAX}`) + // The count is the groups the ranking already forms, counted — not a + // third read of the index for the same aggregation. + expect(sql).toContain("count() AS keyCount") + expect(sql).toContain("AS window_keys") + expect(sql).toContain("'keys' AS period") + }) + + it("ranks the keys the caller asked for, and the table's own cap by default", () => { + expect(breakdownSql({ dimension: "tool", limit: 3 })).toContain("LIMIT 3") + // The cap is the request contract's — a `limit` past it is a 400 and + // never reaches the builder, so nothing re-clamps it here. + expect(breakdownSql({ dimension: "tool" })).toContain(`LIMIT ${AI_OVERVIEW_BREAKDOWN_MAX}`) + }) + + it("groups by the key and by nothing else, so a session counts once per key", () => { + const sql = breakdownSql({ dimension: "model" }) + + expect(sql).toContain("GROUP BY sessionId, key") + expect(sql).toContain("GROUP BY key") + // The totals never group by a key — theirs is the constant every read + // carries so the levels have one shape. + expect(totalsSql()).toContain("GROUP BY sessionId") + expect(totalsSql()).not.toContain("GROUP BY sessionId, key") + }) +}) + +describe("the model mix", () => { + const modelMixSql = (opts: Parameters[0] = {}) => + compileUnsafe(aiOverviewModelMixQuery(opts), seriesParams).sql + + it("counts the model-call spans that name a model, per bucket and band", () => { + const sql = modelMixSql() + + expect(sql).toContain("FROM ai_trace_index") + expect(sql).not.toContain("trace_detail_spans") + expect(sql).not.toContain("__PARAM_") + // The population: model-call spans that named a model. The netting never + // runs here, so a gateway's mirror is a span of its own — the summary's + // `llmCallSpans` population, less the calls that named nothing. + expect(sql).toContain("AND ai_trace_index.IsLlmCall = 1") + expect(sql).toContain("AND ai_trace_index.Model != ''") + expect(sql).toContain("count() AS llmCallSpans") + expect(sql).toContain("GROUP BY bucket, model") + expect(sql).not.toContain("AS netted") + }) + + it("folds every model past the busiest five into one band, in SQL", () => { + const sql = modelMixSql() + + // Ranked once over the window, then read as a band per row: a bucket + // answers at most six rows however many models the org routes across. + expect(sql).toContain("count() AS rankSpans") + expect(sql).toContain("ORDER BY rankSpans DESC, rankModel ASC") + expect(sql).toContain("LIMIT 5) AS top_models") + expect(sql).toContain( + "if(toString(ai_trace_index.Model) IN (SELECT\n rankModel AS topModel", + ) + expect(sql).toContain("'other') AS model") + }) + + it("buckets the span's own timestamp, at the width the caller asked for", () => { + const sql = modelMixSql() + + // The span's timestamp and not the session's start: the rows are spans, + // so there is no session to keep inside one bucket. + expect(sql).toContain("toStartOfInterval(ai_trace_index.Timestamp, INTERVAL 300 SECOND)") + expect(sql).toContain("ORDER BY bucket ASC, llmCallSpans DESC") + // A guard the page cannot reach, now that the tail is folded: ordered by + // bucket, a row cap would have cut the NEWEST buckets off the chart. + expect(sql).toContain(`LIMIT ${AI_OVERVIEW_MODEL_MIX_MAX_ROWS}`) + }) + + it("reads the current window alone, scoped to the org on every level", () => { + const sql = modelMixSql() + + expect(sql).toContain(`Timestamp >= '${params.startTime}'`) + expect(sql).toContain(`Timestamp <= '${params.endTime}'`) + // The chart has no comparison band, so the previous window's params are + // never resolved. + expect(sql).not.toContain(params.prevStartTime) + // The trace rollup and the spans themselves, once for the mix and once + // for the ranking that picks its bands. + expect(orgPredicateCount(sql)).toBe(4) + expect(compileUnsafe(aiOverviewModelMixQuery(), seriesParams).tenantScope).toBe("single-tenant") + }) + + it("selects sessions with the same tests every other overview read applies", () => { + const sql = modelMixSql({ + vendorIds: ["eve"], + models: ["gpt-5.5"], + toolNames: ["send_email"], + hasErrors: true, + }) + + // The per-trace existence tests, so a session that used the model is + // measured across every model it used — and the session-level failure + // test, which adds its own two levels to the org scoping. + expect(sql).toContain("countIf(VendorId IN ('eve')) > 0") + expect(sql).toContain("countIf(Model IN ('gpt-5.5')) > 0") + expect(sql).toContain("countIf(ToolName IN ('send_email')) > 0") + expect(sql).toContain(`${ROLLUP_SESSION_KEY} IN (SELECT`) + expect(orgPredicateCount(sql)).toBe(6) + + const unfiltered = modelMixSql() + expect(unfiltered).not.toContain("HAVING") + expect(unfiltered).not.toContain(`${ROLLUP_SESSION_KEY} IN (SELECT`) + }) +}) diff --git a/packages/query-engine-integrations/src/ai/ai-overview.ts b/packages/query-engine-integrations/src/ai/ai-overview.ts new file mode 100644 index 000000000..f0418f9ce --- /dev/null +++ b/packages/query-engine-integrations/src/ai/ai-overview.ts @@ -0,0 +1,676 @@ +// Agent Sessions › Overview — the warehouse reads behind the overview page. +// +// Everything here is `ai_trace_index` and nothing else (see `ai-sessions.ts` +// for what that index is and what it costs). The page asks three questions of +// one population — what the window totals, how it moved, and where it went — +// and the answer to all three has to be the same numbers the sessions LIST +// shows for the same window, or the two pages describe different products. +// +// That constraint is what shapes the file. Four rules, all borrowed rather +// than re-derived: +// +// 1. A SESSION is a trace-level key: `max(SessionId)` per trace, then +// `sessionKey(…)` from `ai-sessions.ts`, which files a trace whose vendor +// exposes no session key under `trace:`. Counted per row instead, +// every sessionless span collapses into one phantom session. +// 2. A FILTER selects sessions, as a per-trace existence test — "some agent +// span of this trace carries this value" — never a row predicate. The +// three GenAI identity columns are mutually exclusive by construction, so +// a row predicate ANDing a model with a tool can only match a row +// carrying both, and two facets with non-zero counts would return +// nothing. `traceKeys` is that test, and it is the same one `indexTraces` +// makes for the list. +// 3. USAGE is netted per session — `usageReportersExpr` collected over the +// session's spans, then `nettedReportersExpr` and `sessionUsageSum`. +// A wrapper that rolls up its children's tokens, a gateway's second trace +// of the same call, and a provider retry beneath the call each count +// once. A bucketed `sum(Cost)` is one line of SQL and is wrong by +// whatever the org's roll-up rate is. +// 4. A SESSION BELONGS TO ONE BUCKET — the one its first span started in — +// so the series sums to the totals instead of counting a long session in +// every bucket it touched. The session-duration quantiles are the +// exception: they do not merge, which is why the totals are their own +// un-bucketed read rather than a client-side fold of the series. +// +// The breakdown adds a fifth. A key is the value the SPAN ITSELF carries, and +// the netting runs per (session, key): a session that used two models is a +// session under each of them (rows overlap and do not sum to the totals), but +// its tokens are charged to the model whose call reported them rather than +// repeated under both. That is the most faithful per-model attribution the +// reporter mechanism allows without a second netting implementation, and it is +// what keeps a wrapper's roll-up and a gateway's mirror from being counted +// twice inside a key. Its one gap: a reporter whose dimension value differs +// from its children's — an eve turn span rolling up a Vercel model call under a +// `vendor` breakdown — is netted only within its own key, so those two rows +// can together exceed the window's cost. `model` and `tool` are read over the +// spans that can carry them (`IsLlmCall = 1`, `IsToolCall = 1`); the rest are +// read over every agent span, and a span that names no value keys under `''`, +// which the page renders as unattributed rather than hiding. +// +// MODEL CALLS are counted over two populations, because volume and failures +// cannot share one. `llmCalls` is the netted volume: a wrapper's roll-up, a +// gateway's mirror and a provider retry of one call are one call. Failures +// cannot be netted at all — the index carries no error flag into the reporters +// — so `erroredLlmCalls` is a raw `sumIf` over the model-call SPANS, and +// `llmCallSpans` counts exactly those spans so the two divide. Read against +// `llmCalls`, a mirrored call that failed on both observations is two failures +// of one call and the rate passes 100%. +// +// The MODEL MIX is the one read that is a plain GROUP BY over the index, and +// the one that folds its own tail: the models past the busiest few are counted +// under `other` in SQL, so a bucket answers a bounded number of rows however +// many models the org routes across. +// +// Durations stay in NANOSECONDS, like every other AI read — `Duration` is what +// the index stores and the client formats. + +import * as CH from "@maple-dev/effect-clickhouse/expr" +import { + from, + fromQuery, + inSubquery, + param, + unionAll, + type CHQuery, + type CHUnionQuery, + type ColumnDefs, +} from "@maple-dev/effect-clickhouse" +import { AI_OVERVIEW_BREAKDOWN_MAX, type AiOverviewDimension } from "@maple/domain/http" +import { AiTraceIndex } from "@maple/query-engine/ch/tables" +import { finiteOrZero, isoBucket } from "@maple/query-engine/ch/format" +import { sessionFilterConditions, sessionKey } from "./ai-sessions" +import { + childClaimsExpr, + nettedReportersExpr, + reportingSpanIdsExpr, + sessionLlmCalls, + sessionPricedLlmCalls, + sessionUsageSum, + usageReportersExpr, +} from "./ai-span-columns" + +/** + * The page's selection, as every read here takes it — the sessions list's + * counted filters by the same names, so the two pages select the same + * sessions. Each is a per-trace existence test; see rule 2 in the header. + */ +export interface AiOverviewFilterOpts { + readonly vendorIds?: readonly string[] + readonly serviceNames?: readonly string[] + readonly deploymentEnvs?: readonly string[] + readonly models?: readonly string[] + readonly agentNames?: readonly string[] + readonly toolNames?: readonly string[] + /** Sessions with at least one failed agent span — the list's own rule. */ + readonly hasErrors?: boolean +} + +export interface AiOverviewBreakdownOpts extends AiOverviewFilterOpts { + readonly dimension: AiOverviewDimension + /** Keys returned per period. Defaults to {@link AI_OVERVIEW_BREAKDOWN_MAX}, + * which is also where the request contract caps it — a larger `limit` is a + * 400 and never reaches here, so there is nothing to clamp twice. */ + readonly limit?: number +} + +/** + * Which pair of params bounds a read: the caller's window, or the window of + * equal length immediately before it. Both reads use both — one `UNION ALL` + * branch each — and every branch is built from the same expression functions, + * because a `LowCardinality(String)` on one branch against a `String` on + * another is a `NO_COMMON_TYPE`. + */ +type AiOverviewWindow = "current" | "previous" + +/** Which window a row measures. `keys` is the breakdown's third branch: how + * many distinct keys the current window has, before the top-N cut. */ +export type AiOverviewPeriod = "current" | "previous" | "keys" + +/** + * The period a branch stamps on its rows, as a literal the row type keeps. + * + * `CH.lit` widens a string to `string` — most literals are values rather than + * tags — so without this the three-way `period` every caller switches on would + * arrive as an open string. + */ +// SAFETY: the literal compiled into the branch IS the argument, so the row +// carries one of the three tags and nothing else. +const periodLit = (period: AiOverviewPeriod): CH.Expr => + CH.lit(period) as CH.Expr + +const startParam = (window: AiOverviewWindow) => + param.dateTimeString(window === "current" ? "startTime" : "prevStartTime") +const endParam = (window: AiOverviewWindow) => + param.dateTimeString(window === "current" ? "endTime" : "prevEndTime") + +/** + * The window's bounds on a row's timestamp, on every level that reads the + * index. + * + * The caller's window is CLOSED at both ends, the way every other Maple read + * takes one. The comparison window is `[start − length, start)`: it ends where + * the caller's begins, so its upper bound is EXCLUSIVE and a row sitting + * exactly on the boundary belongs to the current window alone rather than to + * both. + * + * The bound is an `Expr` because Maple warehouse timestamps stay the + * strings ClickHouse sends (`tables.ts`), which is what `param.dateTimeString` + * compares against. + */ +const withinWindow = ( + timestamp: CH.Expr, + window: AiOverviewWindow, +): ReadonlyArray => [ + timestamp.gte(startParam(window)), + window === "current" ? timestamp.lte(endParam(window)) : timestamp.lt(endParam(window)), +] + +/** + * One row per agent trace of the window that passes the selection: its id, the + * session it is filed under, and how many of its spans failed. + * + * `max(SessionId)` because the id sits on the turn-owning span alone and every + * other row of the trace reads `''`, which `max` discards. The filters are + * `HAVING countIf(…) > 0` for the same reason `indexTraces` applies them + * there — a row predicate would also narrow the rows the session id is read + * from, and would file a trace under `trace:` whenever its session-bearing + * span belonged to another vendor. + * + * `errorSpans` is read here so the `hasErrors` test below needs no second pass + * over the index: a session failed when the traces filed under it did. + */ +const traceRollup = (opts: AiOverviewFilterOpts, window: AiOverviewWindow) => + from(AiTraceIndex) + .select(($) => ({ + TraceId: $.TraceId, + rawSessionId: CH.max_($.SessionId), + errorSpans: CH.sum($.IsError), + })) + .where(($) => [$.OrgId.eq(param.string("orgId")), ...withinWindow($.Timestamp, window)]) + .groupBy("TraceId") + .having(($) => sessionFilterConditions(opts, $)) + +/** + * The session keys of the window with a failed agent span — the `hasErrors` + * filter, as the list applies it. + * + * A session-level test and not a trace-level one: a session spans traces, and + * the list matches it when ANY of its agent spans failed. Summed over the + * trace rollup, which is the same sum one level up. It reads the whole + * population rather than the dimension's, so a `model` breakdown under + * `hasErrors` measures the sessions the list would have listed. + */ +const erroredSessionKeys = (opts: AiOverviewFilterOpts, window: AiOverviewWindow) => + fromQuery(traceRollup(opts, window), "errored_traces") + .select(($) => ({ sessionId: sessionKey($.rawSessionId, $.TraceId) })) + .groupBy("sessionId") + .having(($) => [CH.sum($.errorSpans).gt(0)]) + +/** + * The trace set every level of every read joins: the window's agent traces + * that passed the selection, keyed by the session they are filed under. + * + * The failed-session test is applied HERE and nowhere else. It is a read of + * its own, and a level that applied it for itself derived the whole errored + * set again — four times over in one breakdown. + */ +const traceKeys = (opts: AiOverviewFilterOpts, window: AiOverviewWindow) => + fromQuery(traceRollup(opts, window), "selected_traces") + .select(($) => ({ TraceId: $.TraceId, rawSessionId: $.rawSessionId })) + .where(($) => [ + CH.whenTrue(opts.hasErrors, () => + inSubquery(sessionKey($.rawSessionId, $.TraceId), erroredSessionKeys(opts, window)), + ), + ]) + +interface CallColumns { + readonly IsLlmCall: CH.Expr + readonly IsToolCall: CH.Expr +} + +/** The spans a dimension's keys can come from. `Model` sits on model calls and + * `ToolName` on tool calls; the rest are properties of every agent span. */ +const dimensionPopulation = ( + dimension: AiOverviewDimension, +): (($: CallColumns) => CH.Condition) | undefined => { + if (dimension === "model") return ($) => $.IsLlmCall.eq(1) + if (dimension === "tool") return ($) => $.IsToolCall.eq(1) + return undefined +} + +/** + * The value a row is filed under, as a plain `String`. + * + * `toString` because four of the six columns are `LowCardinality(String)` and + * the breakdown unions them against a `String` literal on its third branch, + * which is a `NO_COMMON_TYPE` without it. + */ +const dimensionKey = (dimension: AiOverviewDimension) => { + switch (dimension) { + case "model": + return ($: DimensionColumns) => CH.toString_($.Model) + case "agent": + return ($: DimensionColumns) => CH.toString_($.AgentName) + case "service": + return ($: DimensionColumns) => CH.toString_($.ServiceName) + case "environment": + return ($: DimensionColumns) => CH.toString_($.DeploymentEnv) + case "vendor": + return ($: DimensionColumns) => CH.toString_($.VendorId) + case "tool": + return ($: DimensionColumns) => CH.toString_($.ToolName) + } +} + +interface DimensionColumns { + readonly Model: CH.Expr + readonly AgentName: CH.Expr + readonly ServiceName: CH.Expr + readonly DeploymentEnv: CH.Expr + readonly VendorId: CH.Expr + readonly ToolName: CH.Expr +} + +/** + * One row per session (or per session and key), with everything the index + * carries about it: the measures summed over its spans, and its usage still as + * reporters, netted one level up and summed at the grouping level. + * + * `key` is the breakdown's grouping; without it the rows are sessions, which + * is what the totals and the series aggregate. Column names are deliberately + * not the names the levels above select (`sessionStart`, not `bucket`): an + * outer alias shadows a derived column of the same name, and an aggregate over + * the shadowed name becomes a cyclic alias rather than the aggregate meant. + */ +const sessionRows = ( + opts: AiOverviewFilterOpts, + window: AiOverviewWindow, + dimension?: AiOverviewDimension, +) => { + const population = dimension === undefined ? undefined : dimensionPopulation(dimension) + const key = dimension === undefined ? undefined : dimensionKey(dimension) + const rows = from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, window), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + sessionId: sessionKey($.trace.rawSessionId, $.TraceId), + // The totals and the series are a breakdown of one key, so the column + // is always there and is `''` for them — a constant, which needs no + // place in the GROUP BY and costs the read nothing. + key: key === undefined ? CH.lit("") : key($), + // The bucket the session is filed under is cut from this one level + // up — the session's FIRST span, so it lands in exactly one bucket. + sessionStart: CH.min_($.Timestamp), + // `Timestamp` is the span's START, so the extent ends where the + // last-starting span ended. Without the `+ Duration` a session whose + // trace is one long span reports a duration of 0. + sessionDurationNs: CH.max_(CH.toUnixTimestamp64Nano($.Timestamp).add(CH.toInt64($.Duration))).sub( + CH.toUnixTimestamp64Nano(CH.min_($.Timestamp)), + ), + errorSpans: CH.sum($.IsError), + toolCalls: CH.sum($.IsToolCall), + erroredToolCalls: CH.sumIf($.IsError, $.IsToolCall.eq(1)), + // The failed model-call SPANS. Not netted — the index carries no + // error flag into the reporters — so a framework that echoes a + // failure onto the span wrapping the call reports it twice. + erroredLlmCalls: CH.sumIf($.IsError, $.IsLlmCall.eq(1)), + // Its denominator: the SAME spans, counted. The netted `llmCalls` + // below measures a different population — one mirrored call is one + // call there and two failures above — so an error rate taken against + // it can exceed 100%. + llmCallSpans: CH.sum($.IsLlmCall), + // Usage AND model calls travel as reporters: both are counted above, + // where every span of the session is in hand — see `ai-span-columns`. + // The two lookups the netting makes are taken off the reporters here, + // once per session, rather than once per reporter inside the netting. + reporters: usageReportersExpr($), + childClaims: childClaimsExpr("reporters"), + reportingIds: reportingSpanIdsExpr("reporters"), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + ...withinWindow($.Timestamp, window), + population === undefined ? undefined : population($), + ]) + return key === undefined ? rows.groupBy("sessionId") : rows.groupBy("sessionId", "key") +} + +/** The reporters netted into claims — its own level, because the netting reads + * the three columns below it inside lambdas and an alias of the same level + * would be evaluated once per reporter. */ +const nettedRows = (opts: AiOverviewFilterOpts, window: AiOverviewWindow, dimension?: AiOverviewDimension) => + fromQuery(sessionRows(opts, window, dimension), "session_rows").select(($) => ({ + key: $.key, + sessionStart: $.sessionStart, + sessionDurationNs: $.sessionDurationNs, + errorSpans: $.errorSpans, + toolCalls: $.toolCalls, + erroredToolCalls: $.erroredToolCalls, + erroredLlmCalls: $.erroredLlmCalls, + llmCallSpans: $.llmCallSpans, + netted: nettedReportersExpr("reporters", "childClaims", "reportingIds"), + })) + +/** The accessor shape {@link measures} reads off {@link nettedRows}. */ +interface SessionColumns { + readonly sessionDurationNs: CH.Expr + readonly errorSpans: CH.Expr + readonly toolCalls: CH.Expr + readonly erroredToolCalls: CH.Expr + readonly erroredLlmCalls: CH.Expr + readonly llmCallSpans: CH.Expr +} + +/** + * The measures every grouping reports, so a tile, a point on the chart and a + * breakdown row are the same numbers under different `GROUP BY`s. + * + * `count()` rather than `uniqExact`: the level below is already one row per + * session (or per session and key), so a session is counted once and exactly + * once — and a session lands in one bucket, so the series sums to the totals. + * + * The usage sums are the netting evaluated per session and summed over the + * group. Written as one pass per measure over the netted claims, the same + * eight the sessions list makes, because what a level computes is what it + * costs: the warehouse analyses every lambda in a SELECT before it reads a + * row. + */ +const measures = ($: SessionColumns) => ({ + sessions: CH.count(), + erroredSessions: CH.countIf($.errorSpans.gt(0)), + llmCalls: CH.sum(sessionLlmCalls("netted")), + llmCallSpans: CH.sum($.llmCallSpans), + erroredLlmCalls: CH.sum($.erroredLlmCalls), + toolCalls: CH.sum($.toolCalls), + erroredToolCalls: CH.sum($.erroredToolCalls), + cost: CH.sum(sessionUsageSum("netted", "cost")), + pricedLlmCalls: CH.sum(sessionPricedLlmCalls("netted")), + tokens: CH.sum(sessionUsageSum("netted", "tokens")), + inputTokens: CH.sum(sessionUsageSum("netted", "inputTokens")), + cacheReadTokens: CH.sum(sessionUsageSum("netted", "cacheReadTokens")), + cacheWriteTokens: CH.sum(sessionUsageSum("netted", "cacheWriteTokens")), + outputTokens: CH.sum(sessionUsageSum("netted", "outputTokens")), + reasoningTokens: CH.sum(sessionUsageSum("netted", "reasoningTokens")), + // A quantile over an empty group is NULL, which the row schema refuses. + sessionDurationP50Ns: finiteOrZero(CH.quantile(0.5)($.sessionDurationNs)), + sessionDurationP95Ns: finiteOrZero(CH.quantile(0.95)($.sessionDurationNs)), +}) + +/** Every measure at zero — the shape a branch that measures something else + * still has to project, since a `UNION ALL`'s branches share one row. */ +const noMeasures = () => ({ + sessions: CH.lit(0), + erroredSessions: CH.lit(0), + llmCalls: CH.lit(0), + llmCallSpans: CH.lit(0), + erroredLlmCalls: CH.lit(0), + toolCalls: CH.lit(0), + erroredToolCalls: CH.lit(0), + cost: CH.lit(0), + pricedLlmCalls: CH.lit(0), + tokens: CH.lit(0), + inputTokens: CH.lit(0), + cacheReadTokens: CH.lit(0), + cacheWriteTokens: CH.lit(0), + outputTokens: CH.lit(0), + reasoningTokens: CH.lit(0), + sessionDurationP50Ns: CH.lit(0), + sessionDurationP95Ns: CH.lit(0), +}) + +export interface AiOverviewMeasuresOutput { + readonly sessions: number + readonly erroredSessions: number + readonly llmCalls: number + readonly llmCallSpans: number + readonly erroredLlmCalls: number + readonly toolCalls: number + readonly erroredToolCalls: number + readonly cost: number + readonly pricedLlmCalls: number + readonly tokens: number + readonly inputTokens: number + readonly cacheReadTokens: number + readonly cacheWriteTokens: number + readonly outputTokens: number + readonly reasoningTokens: number + readonly sessionDurationP50Ns: number + readonly sessionDurationP95Ns: number +} + +export interface AiOverviewTotalsOutput extends AiOverviewMeasuresOutput { + readonly period: AiOverviewPeriod +} + +export interface AiOverviewSeriesOutput extends AiOverviewTotalsOutput { + /** ISO-8601 with a literal `Z`. */ + readonly bucket: string +} + +export interface AiOverviewBreakdownOutput extends AiOverviewTotalsOutput { + readonly key: string + /** Distinct keys the current window has — carried by the `keys` branch + * alone, 0 on the two that measure. */ + readonly keyCount: number +} + +/** + * The KPI tiles: every measure over the caller's window, and over the window of + * equal length immediately before it. + * + * One read rather than two requests, and not folded from the series either: + * quantiles cannot be merged after the fact, so a p95 for the window is only + * available from a read that grouped the window. The previous branch is bounded + * by its own pair of params (`prevStartTime`/`prevEndTime`), which the caller + * computes — the query has no opinion about what "previous" means beyond + * reading a second window, half-open at its upper bound so a session on the + * boundary is measured once (see {@link withinWindow}). + */ +export function aiOverviewTotalsQuery(opts: AiOverviewFilterOpts = {}): CHUnionQuery { + const branch = (window: AiOverviewWindow) => + fromQuery(nettedRows(opts, window), `netted_${window}`).select(($) => ({ + period: periodLit(window), + ...measures($), + })) + return unionAll(branch("current"), branch("previous")).format("JSON") +} + +/** + * The chart: the same measures, cut into buckets. + * + * A session is filed under the bucket its FIRST span started in, so the points + * sum to the totals — every other reading counts a session that ran across a + * bucket boundary twice. The session's whole netted usage goes with it, which + * is the simplification the attribution makes: at bucket widths of five + * minutes and up a session's spans are inside one bucket or the next. + */ +export function aiOverviewSeriesQuery(opts: AiOverviewFilterOpts = {}): CHUnionQuery { + const branch = (window: AiOverviewWindow) => + fromQuery(nettedRows(opts, window), `netted_${window}`) + .select(($) => ({ + period: periodLit(window), + bucket: isoBucket($.sessionStart), + ...measures($), + })) + .groupBy("bucket") + // Oldest first, so a client plots the points in the order they arrive. + return unionAll(branch("current"), branch("previous")) + .orderBy(["period", "asc"], ["bucket", "asc"]) + .format("JSON") +} + +/** + * Every key of the current window with its session count — one row per key, + * which is both the ranking the table is cut from and the count of what the + * table is not showing. + * + * Ranked on sessions alone, off the raw index rows rather than the netted + * pipeline: which keys the table shows is a question about counts, and running + * the netting a third time to break a tie by cost would cost more than the tie + * is worth. The key breaks ties instead, so two keys with the same session + * count cannot swap places between loads. + */ +const rankedKeys = (opts: AiOverviewBreakdownOpts) => { + const population = dimensionPopulation(opts.dimension) + const key = dimensionKey(opts.dimension) + return from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, "current"), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + rankKey: key($), + rankSessions: CH.uniqExact(sessionKey($.trace.rawSessionId, $.TraceId)), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + ...withinWindow($.Timestamp, "current"), + population === undefined ? undefined : population($), + ]) + .groupBy("rankKey") +} + +/** The busiest of them, as a one-column subquery for `IN`. */ +const topKeys = (opts: AiOverviewBreakdownOpts) => + fromQuery( + rankedKeys(opts) + .orderBy(["rankSessions", "desc"], ["rankKey", "asc"]) + .limit(opts.limit ?? AI_OVERVIEW_BREAKDOWN_MAX), + "top_keys", + ).select(($) => ({ topKey: $.rankKey })) + +/** + * The breakdown table: the busiest keys of the current window, each measured + * over both windows. + * + * Three branches. Two measure the keys the ranking picked — the previous one + * over the same keys, so a key that stopped being used still shows what it + * cost. The third counts the window's distinct keys, which is what lets the + * table say how many it is not showing: the groups the ranking already forms, + * counted, rather than a third read of the index. + */ +export function aiOverviewBreakdownQuery( + opts: AiOverviewBreakdownOpts, +): CHUnionQuery { + // Rendered into BOTH measuring branches: a `CHUnionQuery` takes no `WITH`, + // so the branches have no CTE to share the ranking through. + const keys = topKeys(opts) + const branch = (window: AiOverviewWindow) => + fromQuery(nettedRows(opts, window, opts.dimension), `netted_${window}`) + .select(($) => ({ + period: periodLit(window), + key: $.key, + keyCount: CH.lit(0), + ...measures($), + })) + .where(($) => [inSubquery($.key, keys)]) + .groupBy("key") + const keyCount = fromQuery(rankedKeys(opts), "window_keys").select(() => ({ + period: periodLit("keys"), + key: CH.lit(""), + keyCount: CH.count(), + ...noMeasures(), + })) + return unionAll(branch("current"), branch("previous"), keyCount).format("JSON") +} + +/** Models the mix plots as bands of their own. Everything past them is one + * `other` band, which is what bounds the response. */ +const AI_OVERVIEW_MODEL_MIX_BANDS = 5 + +/** The band every model outside the top {@link AI_OVERVIEW_MODEL_MIX_BANDS} is + * counted under — the key the client folds its own tail into. */ +const AI_OVERVIEW_MODEL_MIX_OTHER = "other" + +/** + * Rows one model mix returns, across every bucket and band together. + * + * A guard the page cannot reach rather than a cut: the tail is folded in SQL, + * so a bucket answers at most six rows and a year of daily buckets is still + * well inside this. + */ +export const AI_OVERVIEW_MODEL_MIX_MAX_ROWS = 4000 + +export interface AiOverviewModelMixOutput { + /** ISO-8601 with a literal `Z`. */ + readonly bucket: string + /** The model the spans named, or the folded `other` band. */ + readonly model: string + readonly llmCallSpans: number +} + +/** + * The busiest models of the window, as a one-column subquery for `IN`. + * + * Ranked over the population the mix counts — model-call SPANS naming a model, + * among the selected sessions — so the bands are the ones a full answer would + * have shown. The name breaks ties, so two models with the same count cannot + * swap bands between loads. + */ +const topModels = (opts: AiOverviewFilterOpts) => { + const ranked = from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, "current"), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ rankModel: CH.toString_($.Model), rankSpans: CH.count() })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + ...withinWindow($.Timestamp, "current"), + $.IsLlmCall.eq(1), + $.Model.neq(""), + ]) + .groupBy("rankModel") + .orderBy(["rankSpans", "desc"], ["rankModel", "asc"]) + .limit(AI_OVERVIEW_MODEL_MIX_BANDS) + return fromQuery(ranked, "top_models").select(($) => ({ topModel: $.rankModel })) +} + +/** + * The model mix: the window's model-call SPANS, split by band, bucket by + * bucket. + * + * The share of model SPANS and not of netted calls — this is a plain GROUP BY + * over the index, where the netting is a per-session array pass — so a + * gateway's mirror of a call is counted under the model it names, twice. It is + * the same population the summary counts as `llmCallSpans`, less the calls + * whose instrumentation named no model: those carry no share of a model mix, + * so the two totals differ by exactly them. + * + * The TAIL IS FOLDED HERE. Ordered by bucket and cut at a row cap, the cap + * drops the newest buckets — the end of the chart — in exactly the org that + * needs the chart most. Ranking the window's models once and counting the rest + * under `other` bounds a bucket at six rows instead, and it is the band the + * client would have folded anyway. + * + * A span is filed under the bucket ITS OWN timestamp falls in, where the + * summary's series files a whole session under the bucket it started in. The + * rows here are spans, so there is no session to keep whole. + * + * Sessions are selected the way every other read in this file selects them — + * `traceKeys`, the failed-session test included — so the mix describes the + * sessions the tiles above it measure. The current window alone: the chart has + * no comparison band. + */ +export function aiOverviewModelMixQuery( + opts: AiOverviewFilterOpts = {}, +): CHQuery { + const bands = topModels(opts) + return from(AiTraceIndex) + .innerJoinQuery(traceKeys(opts, "current"), "trace", (row, trace) => row.TraceId.eq(trace.TraceId)) + .select(($) => ({ + bucket: isoBucket($.Timestamp), + // `toString` for the reason the breakdown's key takes it: `Model` is + // `LowCardinality(String)` in the index, and a model key is a plain + // `String` everywhere else the page reads one. + model: CH.if_( + inSubquery(CH.toString_($.Model), bands), + CH.toString_($.Model), + CH.lit(AI_OVERVIEW_MODEL_MIX_OTHER), + ), + llmCallSpans: CH.count(), + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + ...withinWindow($.Timestamp, "current"), + $.IsLlmCall.eq(1), + $.Model.neq(""), + ]) + .groupBy("bucket", "model") + .orderBy(["bucket", "asc"], ["llmCallSpans", "desc"]) + .limit(AI_OVERVIEW_MODEL_MIX_MAX_ROWS) + .format("JSON") +} diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index fe0fdf1aa..df8c1ac07 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -237,7 +237,7 @@ const orderTuple = (...parts: ReadonlyArray): CH.Expr => * a session-bearing trace carry no session id themselves, and keying on that * would file each of them as its own sessionless trace. */ -const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr): CH.Expr => +export const sessionKey = (rawSessionId: CH.Expr, traceId: CH.Expr): CH.Expr => CH.if_(rawSessionId.eq(""), CH.concat(MAPLE_AI_TRACE_SESSION_PREFIX, traceId), rawSessionId) /** @@ -317,6 +317,38 @@ export interface AiSessionFilterOpts { readonly search?: string } +/** + * The six counted filters, as every read that selects sessions applies them. + * + * A TRACE-level existence test — "some agent span of this trace carries this + * value" — and never a row predicate, for the reasons {@link indexTraces} + * gives. Shared with the overview's `traceKeys` so the list and the overview + * cannot drift into selecting different sessions; the list appends its own + * `search` clause after these. + */ +export const sessionFilterConditions = ( + opts: AiSessionFilterOpts, + $: { + readonly VendorId: CH.Expr + readonly ServiceName: CH.Expr + readonly DeploymentEnv: CH.Expr + readonly Model: CH.Expr + readonly AgentName: CH.Expr + readonly ToolName: CH.Expr + }, +) => { + const values = (list: readonly string[] | undefined) => (list?.length ? list : undefined) + const carries = (cond: CH.Condition) => CH.countIf(cond).gt(0) + return [ + CH.when(values(opts.vendorIds), (v) => carries(CH.inList($.VendorId, v))), + CH.when(values(opts.serviceNames), (v) => carries(CH.inList($.ServiceName, v))), + CH.when(values(opts.deploymentEnvs), (v) => carries(CH.inList($.DeploymentEnv, v))), + CH.when(values(opts.models), (v) => carries(CH.inList($.Model, v))), + CH.when(values(opts.agentNames), (v) => carries(CH.inList($.AgentName, v))), + CH.when(values(opts.toolNames), (v) => carries(CH.inList($.ToolName, v))), + ] +} + export interface AiSessionPageOpts extends AiSessionFilterOpts { /** Sessions returned, most recently started first unless `sortBy` says otherwise. */ readonly limit?: number @@ -488,9 +520,7 @@ const MAX_NAMES_PER_TRACE = 20 * `usageReportersExpr`. */ const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds) => { - const values = (list: readonly string[] | undefined) => (list?.length ? list : undefined) const search = opts.search?.trim() || undefined - const carries = (cond: CH.Condition) => CH.countIf(cond).gt(0) return from(AiTraceIndex) .select(($) => { // Ranks the trace's spans for the agent-name `argMin`: a span that @@ -564,15 +594,10 @@ const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds) => { ]) .groupBy("traceId") .having(($) => [ - CH.when(values(opts.vendorIds), (v) => carries(CH.inList($.VendorId, v))), - CH.when(values(opts.serviceNames), (v) => carries(CH.inList($.ServiceName, v))), - CH.when(values(opts.deploymentEnvs), (v) => carries(CH.inList($.DeploymentEnv, v))), - CH.when(values(opts.models), (v) => carries(CH.inList($.Model, v))), - CH.when(values(opts.agentNames), (v) => carries(CH.inList($.AgentName, v))), - CH.when(values(opts.toolNames), (v) => carries(CH.inList($.ToolName, v))), + ...sessionFilterConditions(opts, $), CH.when(search, (needle) => { const pattern = idSearchPattern(needle) - return carries($.SessionId.like(pattern).or($.TraceId.like(pattern))) + return CH.countIf($.SessionId.like(pattern).or($.TraceId.like(pattern))).gt(0) }), ]) } diff --git a/packages/query-engine-integrations/src/ai/ai-span-columns.ts b/packages/query-engine-integrations/src/ai/ai-span-columns.ts index d7f6b817e..9e5fd0319 100644 --- a/packages/query-engine-integrations/src/ai/ai-span-columns.ts +++ b/packages/query-engine-integrations/src/ai/ai-span-columns.ts @@ -211,3 +211,24 @@ export function sessionUsageSum(netted: string, measure: SessionUsageMeasure): E export function sessionLlmCalls(netted: string): Expr { return CH.rawExpr(`toFloat64(${nettedSum(netted, 2, "toFloat64(n.2)")})`, T.float64) } + +/** + * The session's model calls that carried a PRICE — {@link sessionLlmCalls} + * restricted to the reporters whose netted cost is above zero. + * + * The coverage behind a cost figure, and it has to be netted to be a share of + * anything: `Cost` is whatever the instrumentation reported and nothing prices + * a call Maple-side, so a window's cost is only as complete as the calls that + * carried one — and a gateway that prices the call the app's SDK could not is + * the same call twice until the response id collapses it. + * + * Written out rather than passed through {@link nettedSum}, whose unkeyed half + * reads one element of the tuple: this claim is a condition over two of them. + */ +export function sessionPricedLlmCalls(netted: string): Expr { + const priced = "toFloat64(n.2 AND n.4 > 0)" + return CH.rawExpr( + `arraySum(arrayMap(n -> ${priced}, arrayFilter(n -> n.1 = '', ${netted}))) + arraySum(mapValues(arrayReduce('maxMap', arrayMap(n -> map(n.1, ${priced}), arrayFilter(n -> n.1 != '', ${netted})))))`, + T.float64, + ) +} diff --git a/packages/query-engine-integrations/src/ai/index.ts b/packages/query-engine-integrations/src/ai/index.ts index 6acc7ee69..43541de11 100644 --- a/packages/query-engine-integrations/src/ai/index.ts +++ b/packages/query-engine-integrations/src/ai/index.ts @@ -40,6 +40,22 @@ export { type AiSessionWindowOutput, } from "./ai-sessions" +export { + aiOverviewBreakdownQuery, + aiOverviewModelMixQuery, + aiOverviewSeriesQuery, + aiOverviewTotalsQuery, + AI_OVERVIEW_MODEL_MIX_MAX_ROWS, + type AiOverviewBreakdownOpts, + type AiOverviewBreakdownOutput, + type AiOverviewFilterOpts, + type AiOverviewMeasuresOutput, + type AiOverviewModelMixOutput, + type AiOverviewPeriod, + type AiOverviewSeriesOutput, + type AiOverviewTotalsOutput, +} from "./ai-overview" + export { aiFieldSourceKeys, aiSpanAttributeKeys, diff --git a/packages/query-engine-integrations/src/benchmark/index.ts b/packages/query-engine-integrations/src/benchmark/index.ts index 980c0ae8d..79e052569 100644 --- a/packages/query-engine-integrations/src/benchmark/index.ts +++ b/packages/query-engine-integrations/src/benchmark/index.ts @@ -64,6 +64,13 @@ const traceWindow = { * because the page was ranked inside it. */ const AI_PAGE_SESSION_IDS = ["wrun_sql_catalog", `${MAPLE_AI_TRACE_SESSION_PREFIX}${AI_TRACE_ID}`] +/** The overview's reads see two windows at once: the caller's, and the one of + * equal length immediately before it that the tiles compare against. */ +const aiCompare = { ...window, prevStartTime: "2025-12-30 06:45:00", prevEndTime: START_TIME } + +/** The same, plus the bucket the chart is cut at. */ +const aiCompareBucketed = { ...aiCompare, bucketSeconds: 300 } + /** Stage two's whole param set — it never sees the caller's window: the * page's bounds for the index levels, and one slice of the padded extent * (`aiSessionDetailsSlices`) for the fan-out. */ @@ -190,6 +197,79 @@ export const integrationFixtures: ReadonlyArray = [ label: "default", compile: () => compileUnionUnsafe(CH.aiSessionFacetsQuery(), window), }, + { + // The overview's tiles: every measure over the window and over the one + // before it, in one read. The netting runs inside an aggregate here, + // which is a shape no other builder emits. + module: "ai-overview", + name: "aiOverviewTotalsQuery", + label: "default", + compile: () => compileUnionUnsafe(CH.aiOverviewTotalsQuery(), aiCompare), + }, + { + // Every filter the sidebar can send at once: the per-trace existence + // tests, plus the session-level `hasErrors` subquery, which is its own + // SQL shape. + module: "ai-overview", + name: "aiOverviewTotalsQuery", + label: "every-filter", + compile: () => + compileUnionUnsafe( + CH.aiOverviewTotalsQuery({ + vendorIds: ["eve"], + serviceNames: ["maple-slack-agent"], + deploymentEnvs: ["production"], + models: ["gpt-5.5"], + agentNames: ["billing-agent"], + toolNames: ["send_email"], + hasErrors: true, + }), + aiCompare, + ), + }, + { + module: "ai-overview", + name: "aiOverviewSeriesQuery", + label: "default", + compile: () => compileUnionUnsafe(CH.aiOverviewSeriesQuery(), aiCompareBucketed), + }, + { + // A model breakdown reads model calls alone and keys off `Model`. + module: "ai-overview", + name: "aiOverviewBreakdownQuery", + label: "model", + compile: () => + compileUnionUnsafe(CH.aiOverviewBreakdownQuery({ dimension: "model" }), aiCompare), + }, + { + // A tool breakdown reads tool calls alone and keys off `ToolName`. + module: "ai-overview", + name: "aiOverviewBreakdownQuery", + label: "tool", + compile: () => + compileUnionUnsafe(CH.aiOverviewBreakdownQuery({ dimension: "tool", limit: 5 }), aiCompare), + }, + { + // The other four dimensions share one shape: every agent span, keyed by + // a column the span always carries. + module: "ai-overview", + name: "aiOverviewBreakdownQuery", + label: "service", + compile: () => + compileUnionUnsafe( + CH.aiOverviewBreakdownQuery({ dimension: "service", hasErrors: true }), + aiCompare, + ), + }, + { + // The model mix: model-call SPANS per bucket and model, off the same + // selection and with no netting at all — the one overview read that is a + // plain GROUP BY over the index. + module: "ai-overview", + name: "aiOverviewModelMixQuery", + label: "default", + compile: () => compileUnsafe(CH.aiOverviewModelMixQuery(), bucketed), + }, { module: "ai-sessions", name: "aiSessionSpansQuery",