Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/ai/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@
"@effect/ai-openai-compat": "catalog:effect",
"@effect/ai-openrouter": "catalog:effect",
"@maple-dev/effect-sdk": "workspace:*",
"@maple/agent-sessions": "workspace:*",
"@maple/backend": "workspace:*",
"@maple/db": "workspace:*",
"@maple/domain": "workspace:*",
"@maple/infra": "workspace:*",
"@maple/query-engine": "workspace:*",
"@maple/query-engine-integrations": "workspace:*",
"@maple/query-model": "workspace:*",
"@maple/widgets": "workspace:*",
"drizzle-orm": "^0.45.1",
Expand Down
10 changes: 10 additions & 0 deletions apps/ai/src/mcp/__evals__/eval-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,13 @@ export const runToolDirect = async (
McpToolExecutor.pipe(Effect.flatMap((executor) => executor.execute(rt.tenant, name, params, "mcp"))),
)
}

/**
* The markdown a tool rendered — the FIRST text block only. `createDualContent`
* writes the `__maple_ui` mirror as a second block, and joining both lets a
* markdown assertion pass on the structured payload alone.
*/
export const markdown = (result: unknown): string => {
const content = (result as { readonly content?: ReadonlyArray<{ readonly text?: string }> }).content
return content?.[0]?.text ?? ""
}
11 changes: 4 additions & 7 deletions apps/ai/src/mcp/__evals__/execution.eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { describeMapleEval, FIXTURES } from "./utils"
import { createEvalModel, hasEvalCredentials } from "./model"
import { buildExecutionToolSet } from "./tools"
import { installFakeWarehouse, restoreWarehouse } from "./fake-warehouse"
import { makeEvalRuntime, type EvalRuntime } from "./eval-runtime"
import { makeEvalRuntime, markdown, type EvalRuntime } from "./eval-runtime"
import { OutputContainsScorer } from "./scorers"
import { LARGE_TRACE_SPAN_COUNT } from "./fixtures"

Expand All @@ -25,12 +25,9 @@ afterAll(async () => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const extractText = (toolResult: any): string => {
const out = toolResult?.output ?? toolResult?.result
const content = out?.content
if (Array.isArray(content)) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
return content.map((c: any) => c?.text ?? "").join("\n")
}
return typeof out === "string" ? out : ""
// `markdown` takes the FIRST text block: the `__maple_ui` mirror a dual-content
// tool writes as a second block would otherwise satisfy a markdown assertion.
return typeof out === "string" ? out : markdown(out)
}

// Full-execution eval: the model actually calls inspect_trace, which runs end
Expand Down
23 changes: 20 additions & 3 deletions apps/ai/src/mcp/__evals__/fake-warehouse.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Effect } from "effect"
import { WarehouseDriverError } from "@maple/query-engine/execution"
import { WarehouseDriverError, WarehouseResponseLimitError } from "@maple/query-engine/execution"
import { __testables } from "@maple/backend/services/warehouse/WarehouseQueryService"
import { makeLargeTraceSpans, makeTraceLogs } from "./fixtures"

Expand All @@ -25,12 +25,29 @@ const defaultTraceFixtures = (): FixtureRule[] => [
* pipe-dispatch, row parsing) — only the wire call is faked. Unmatched SQL
* throws loudly so missing fixtures never look like an empty result.
*/
export const installFakeWarehouse = (rules: FixtureRule[] = defaultTraceFixtures()): void => {
export const installFakeWarehouse = (
rules: FixtureRule[] = defaultTraceFixtures(),
/** SQL the warehouse aborts on rather than answering with rows. The client is
* cached for the runtime's lifetime, so a test switches the failure from
* inside this hook rather than by re-installing. */
failWhen?: (sql: string) => boolean,
): void => {
__testables.setClientFactory(() =>
Effect.succeed({
sql: (statement) =>
Effect.suspend(() => {
Effect.suspend((): Effect.Effect<
{ data: ReadonlyArray<Record<string, unknown>> },
WarehouseDriverError | WarehouseResponseLimitError
> => {
const sql = statement.text
if (failWhen?.(sql) === true) {
return Effect.fail(
new WarehouseResponseLimitError({
kind: "bytes",
message: "response exceeded the byte limit",
}),
)
}
const rule = rules.find((r) => r.match(sql))
if (!rule) {
return Effect.fail(
Expand Down
126 changes: 126 additions & 0 deletions apps/ai/src/mcp/__evals__/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,132 @@ export const makeSpanDetailRows = (): ReadonlyArray<Record<string, unknown>> =>
},
]

/* -------------------------------------------------------------------------- */
/* An AI agent span, as `inspect_span` reaches it: the point lookup that finds */
/* it, and the trace read that decodes it */
/* -------------------------------------------------------------------------- */

export const AI_SPAN_TRACE_ID = "4d2c1b0a9f8e7d6c5b4a3928170615ff"
/** The LLM call: it requests a tool, whose result is on the span below. */
export const AI_SPAN_SPAN_ID = "a1a1a1a1a1a1a1a1"
export const AI_TOOL_SPAN_ID = "a2a2a2a2a2a2a2a2"
/** A trace too large for one read, and an AI span past its first page. */
export const PARTIAL_AI_TRACE_ID = "5e3d2c1b0a9f8e7d6c5b4a3928170611"
export const PARTIAL_AI_SPAN_ID = "b9b9b9b9b9b9b9b9"
export const PARTIAL_AI_TRACE_SPANS = 2_001
/** The same trace's FIRST span: decoded, but with the rest of the trace — and
* so the tool span answering its call — past what the read carried. */
export const PARTIAL_AI_FIRST_SPAN_ID = "c000000000000000"

const AI_SPAN_ATTRIBUTES = {
"maple_ai.session.id": "wrun_01KZEVAL",
"maple_ai.vendor.id": "eve",
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "gpt-5",
"gen_ai.response.model": "gpt-5",
"gen_ai.usage.input_tokens": "1200",
"gen_ai.input.messages": JSON.stringify([
{ role: "user", parts: [{ type: "text", content: "why is checkout failing?" }] },
]),
"gen_ai.output.messages": JSON.stringify([
{
role: "assistant",
parts: [{ type: "tool_call", id: "call_1", name: "run_sql", arguments: { sql: "select 1" } }],
},
]),
}

/** The point lookup, in `spanDetailQuery`'s output shape. */
export const makeAiSpanDetailRows = (
spanId: string = AI_SPAN_SPAN_ID,
traceId: string = AI_SPAN_TRACE_ID,
): ReadonlyArray<Record<string, unknown>> => [
{
traceId,
spanId,
parentSpanId: "",
spanName: "chat gpt-5",
serviceName: FIXTURES.service,
spanKind: "Client",
durationMs: 1_200,
startTime: "2026-06-02 10:00:00",
statusCode: "Unset",
statusMessage: "",
spanAttributes: JSON.stringify(AI_SPAN_ATTRIBUTES),
resourceAttributes: JSON.stringify({ "service.name": FIXTURES.service }),
},
]

/** One agent span in the wire shape `aiSessionSpansRowSchema` decodes — the row
* every AI read (a session's spans, a trace's spans) answers with. */
export const aiSpanRow = (row: {
readonly spanId: string
readonly spanName: string
readonly attributes: Record<string, string>
readonly traceId?: string
readonly parentSpanId?: string
readonly timestamp?: string
readonly durationMs?: number
readonly statusCode?: string
}): Record<string, unknown> => ({
traceId: row.traceId ?? AI_SPAN_TRACE_ID,
spanId: row.spanId,
parentSpanId: row.parentSpanId ?? "",
spanName: row.spanName,
spanKind: "Client",
serviceName: FIXTURES.service,
durationMs: row.durationMs ?? 1_200,
statusCode: row.statusCode ?? "Unset",
statusMessage: row.statusCode === "Error" ? "tool call failed" : "",
timestamp: row.timestamp ?? "2026-06-02 10:00:00.000000000",
spanAttributes: row.attributes,
})

/** The trace behind the AI span, in `aiSessionSpansRowSchema`'s shape: the call
* and the tool span that answered it. */
export const makeAiTraceSpanRows = (): ReadonlyArray<Record<string, unknown>> => [
aiSpanRow({ spanId: AI_SPAN_SPAN_ID, spanName: "chat gpt-5", attributes: AI_SPAN_ATTRIBUTES }),
aiSpanRow({
spanId: AI_TOOL_SPAN_ID,
spanName: "execute_tool run_sql",
attributes: {
"maple_ai.session.id": "wrun_01KZEVAL",
"maple_ai.vendor.id": "eve",
"gen_ai.operation.name": "execute_tool",
"gen_ai.tool.name": "run_sql",
"gen_ai.tool.call.id": "call_1",
"gen_ai.tool.call.result": JSON.stringify({ error: "table orders does not exist" }),
},
}),
]

/** One row past the trace-pinned read's limit, and the inspected span in none
* of them: the read returns a cursor, so absence is not absence. */
export const makePartialAiTraceSpanRows = (): ReadonlyArray<Record<string, unknown>> =>
Array.from({ length: PARTIAL_AI_TRACE_SPANS }, (_, index) =>
aiSpanRow({
spanId: `c${index.toString(16).padStart(15, "0")}`,
spanName: "chat gpt-5",
// The first span is the LLM call that requested `call_1`; the tool span
// that answered it is not in this trace's first page.
attributes:
index === 0
? AI_SPAN_ATTRIBUTES
: { "maple_ai.vendor.id": "eve", "gen_ai.operation.name": "chat" },
traceId: PARTIAL_AI_TRACE_ID,
}),
)

/** A trace's own bounds, as `aiTraceWindowQuery` reports them: what
* `inspect_span` resolves before it reads the trace as agent spans. */
export const makeAiTraceWindowRows = (): ReadonlyArray<Record<string, unknown>> => [
{
startTime: "2026-06-01 10:00:00.000000000",
endTime: "2026-06-03 10:00:00.000000000",
spanCount: PARTIAL_AI_TRACE_SPANS,
},
]

export const makeTraceLogs = (): ListLogsOutput[] => [
{
timestamp: "2026-06-02 10:00:00",
Expand Down
Loading