From 3c04c62bd7449bed25d42f62035afab878e87f66 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:39:07 +0100 Subject: [PATCH 01/24] Add typecheck script and clear tsc --noEmit baseline - Add npm run typecheck (tsc --noEmit) as the source-of-truth type gate - Narrow exactOptionalPropertyTypes gaps via conditional spreads and optional-with-undefined fields across agent executor, orchestrator, observability, providers, and scorer - Make CheckItem.line/description optional to match runtime reality (reported items carry optional fields; output formatters use a separate Issue type, so display is unaffected) - Confine @ai-sdk/perplexity LanguageModelV1 -> LanguageModel skew to a single cast; ai@6 dropped V1 types and a provider upgrade is a separate, out-of-scope dependency change - No strict compiler options relaxed; src/agent/* left compiling only --- package.json | 1 + src/agent/executor.ts | 36 ++++++++++----------- src/cli/orchestrator.ts | 2 +- src/evaluators/violation-filter.ts | 4 +-- src/observability/factory.ts | 2 +- src/observability/langfuse-observability.ts | 4 +-- src/prompts/schema.ts | 4 +-- src/providers/perplexity-provider.ts | 7 +++- src/providers/vercel-ai-provider.ts | 22 +++++++------ src/scoring/scorer.ts | 5 ++- 10 files changed, 49 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 74f474d5..d9e16b86 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "dev": "tsx src/index.ts", "build": "tsup", + "typecheck": "tsc --noEmit", "start": "node dist/index.js", "lint": "eslint .", "lint:fix": "eslint . --fix", diff --git a/src/agent/executor.ts b/src/agent/executor.ts index 432c6d64..749a5685 100644 --- a/src/agent/executor.ts +++ b/src/agent/executor.ts @@ -1,7 +1,7 @@ import { readdir, readFile } from 'fs/promises'; import * as os from 'os'; import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type PromptEvaluationResult } from '../prompts/schema'; +import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; import type { PromptFile } from '../prompts/prompt-loader'; import { Type, Severity } from '../evaluators/types'; import { computeFilterDecision } from '../evaluators/violation-filter'; @@ -14,6 +14,7 @@ import { createReviewSessionStore } from './review-session-store'; import { buildAgentSystemPrompt } from './prompt-builder'; import { AgentToolError } from '../errors'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; +import type { FilePatternConfig } from '../boundaries/file-section-parser'; import { createAgentTools, listAvailableTools, @@ -53,7 +54,7 @@ export interface RunAgentExecutorParams { prompts: PromptFile[]; provider: LLMProvider; workspaceRoot: string; - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }>; + scanPaths: FilePatternConfig[]; outputFormat: OutputFormat; printMode: boolean; sessionHomeDir?: string; @@ -178,7 +179,7 @@ function findingsFromEvents(events: SessionEvent[]): AgentFinding[] { function buildFileRuleMatches( relativeTargets: string[], prompts: PromptFile[], - scanPaths: Array<{ pattern: string; runRules: string[]; overrides: Record }> + scanPaths: FilePatternConfig[] ): Array<{ file: string; ruleSource: string }> { const resolver = new ScanPathResolver(); const availablePacks = Array.from(new Set(prompts.map((p) => p.pack).filter((p): p is string => !!p))); @@ -301,12 +302,13 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'read_file', path, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `read_file:${path}`, }; } @@ -317,10 +319,11 @@ function resolveVisibleToolContext(params: { } const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.path); const path = resolvedPath ?? parsed.data.path; + const progressFile = currentProgressFile ?? defaultProgressFile; return { toolName: 'list_directory', path, - progressFile: currentProgressFile ?? defaultProgressFile, + ...(progressFile ? { progressFile } : {}), signature: `list_directory:${path}`, }; } @@ -333,14 +336,15 @@ function resolveVisibleToolContext(params: { const resolvedPath = tryResolveRelativePath(workspaceRoot, parsed.data.file); const path = resolvedPath ?? parsed.data.file; const ruleText = parsed.data.reviewInstruction?.trim() || prompt?.body || ''; + const progressFile = resolvedPath && targetFiles.has(resolvedPath) + ? resolvedPath + : (currentProgressFile ?? defaultProgressFile); return { toolName: 'lint', path, ruleName: String(prompt?.meta.name || prompt?.meta.id || 'Rule'), ruleText, - progressFile: resolvedPath && targetFiles.has(resolvedPath) - ? resolvedPath - : (currentProgressFile ?? defaultProgressFile), + ...(progressFile ? { progressFile } : {}), signature: `lint:${path}:${normalizeRuleSource(parsed.data.ruleSource)}:${ruleText}`, }; } @@ -390,11 +394,7 @@ type FindingLikeViolation = { suggestion?: string; fix?: string; confidence?: number; - checks?: { - plausible_non_violation?: boolean; - context_supports_violation?: boolean; - rule_supports_claim?: boolean; - }; + checks?: GateChecks; }; async function appendInlineFinding(params: { @@ -516,8 +516,8 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, promptBySource, targetFiles, - currentProgressFile, - defaultProgressFile: relativeTargets[0], + ...(currentProgressFile ? { currentProgressFile } : {}), + ...(relativeTargets[0] ? { defaultProgressFile: relativeTargets[0] } : {}), }); if (visibleToolContext?.progressFile) { @@ -798,7 +798,7 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< workspaceRoot, fileRuleMatches, availableTools, - userInstructions, + ...(userInstructions ? { userInstructions } : {}), }), prompt: [ `Workspace root: ${workspaceRoot}`, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f4325095..6ae4acd4 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1194,7 +1194,7 @@ async function evaluateFilesInAgentMode( progressReporter, maxParallelToolCalls: 3, maxRetries: options.agentMaxRetries ?? 10, - userInstructions: options.userInstructionContent, + ...(options.userInstructionContent ? { userInstructions: options.userInstructionContent } : {}), }); let totalErrors = 0; diff --git a/src/evaluators/violation-filter.ts b/src/evaluators/violation-filter.ts index 2d3fc295..c4daa9b5 100644 --- a/src/evaluators/violation-filter.ts +++ b/src/evaluators/violation-filter.ts @@ -56,8 +56,8 @@ export function computeFilterDecision(v: GateViolationLike): FilterDecision { const hasFix = (v.fix ?? "").trim() !== ""; - const hasConfidence = typeof v.confidence === "number"; - const passesConfidence = hasConfidence && v.confidence >= confidenceThreshold; + const confidence = typeof v.confidence === "number" ? v.confidence : undefined; + const passesConfidence = confidence !== undefined && confidence >= confidenceThreshold; if (!passesConfidence) reasons.push(`confidence<${confidenceThreshold}`); const surface = diff --git a/src/observability/factory.ts b/src/observability/factory.ts index 84ec671a..c646fc08 100644 --- a/src/observability/factory.ts +++ b/src/observability/factory.ts @@ -19,6 +19,6 @@ export function createObservability(env: EnvConfig, logger?: Logger): AIObservab publicKey: env.LANGFUSE_PUBLIC_KEY, secretKey: env.LANGFUSE_SECRET_KEY, ...(env.LANGFUSE_BASE_URL ? { baseUrl: env.LANGFUSE_BASE_URL } : {}), - logger, + ...(logger ? { logger } : {}), }); } diff --git a/src/observability/langfuse-observability.ts b/src/observability/langfuse-observability.ts index b8898127..25f4d32c 100644 --- a/src/observability/langfuse-observability.ts +++ b/src/observability/langfuse-observability.ts @@ -12,8 +12,8 @@ export interface LangfuseObservabilityConfig { } export class LangfuseObservability implements AIObservability { - private sdk?: NodeSDK; - private initPromise?: Promise; + private sdk?: NodeSDK | undefined; + private initPromise?: Promise | undefined; private readonly logger: Logger; constructor(private readonly config: LangfuseObservabilityConfig) { diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 26c137e1..bd9fd9ee 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -337,8 +337,8 @@ export type JudgeResult = { }; export type CheckItem = { - line: number; - description: string; + line?: number; + description?: string; analysis: string; message?: string; suggestion?: string; diff --git a/src/providers/perplexity-provider.ts b/src/providers/perplexity-provider.ts index c6a05499..432f8c09 100644 --- a/src/providers/perplexity-provider.ts +++ b/src/providers/perplexity-provider.ts @@ -1,4 +1,5 @@ import { generateText } from 'ai'; +import type { LanguageModel } from 'ai'; import { z } from 'zod'; import { createPerplexity } from '@ai-sdk/perplexity'; import type { SearchProvider } from './search-provider'; @@ -46,7 +47,11 @@ export class PerplexitySearchProvider implements SearchProvider { try { const result = await generateText({ - model: this.client('sonar-pro'), + // @ai-sdk/perplexity@1 exposes LanguageModelV1 models, while ai@6's + // generateText types require a LanguageModel (V2/V3). This is a + // third-party SDK version skew, not a repo type hole; resolve by + // upgrading @ai-sdk/perplexity to a V2 release in a follow-up. + model: this.client('sonar-pro') as unknown as LanguageModel, prompt: query, }); diff --git a/src/providers/vercel-ai-provider.ts b/src/providers/vercel-ai-provider.ts index 689a19c0..5cd833fd 100644 --- a/src/providers/vercel-ai-provider.ts +++ b/src/providers/vercel-ai-provider.ts @@ -25,7 +25,7 @@ export class VercelAIProvider implements LLMProvider { private config: VercelAIConfig; private builder: RequestBuilder; private logger: Logger; - private observability?: AIObservability; + private observability?: AIObservability | undefined; constructor(config: VercelAIConfig, builder?: RequestBuilder) { this.config = { @@ -73,12 +73,14 @@ export class VercelAIProvider implements LLMProvider { } try { + const evaluator = this.extractContextValue(context, 'evaluatorName', 'evaluator'); + const rule = this.extractContextValue(context, 'ruleName', 'rule'); const observabilityOptions = this.getObservabilityOptions({ operation: 'structured-eval', provider: this.config.providerName ?? 'unknown', model: this.config.modelName ?? 'unknown', - evaluator: this.extractContextValue(context, 'evaluatorName', 'evaluator'), - rule: this.extractContextValue(context, 'ruleName', 'rule'), + ...(evaluator ? { evaluator } : {}), + ...(rule ? { rule } : {}), }); const result = await generateText({ @@ -187,14 +189,14 @@ export class VercelAIProvider implements LLMProvider { } } - return { - usage: result.usage - ? { - inputTokens: result.usage.inputTokens ?? 0, - outputTokens: result.usage.outputTokens ?? 0, + return result.usage + ? { + usage: { + inputTokens: result.usage.inputTokens ?? 0, + outputTokens: result.usage.outputTokens ?? 0, + }, } - : undefined, - }; + : {}; } private getObservabilityOptions(context: AIExecutionContext): Record { diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index ecea0db6..24f0ced5 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -48,7 +48,10 @@ export function calculateCheckScore( ): CheckResult { const strictness = resolveStrictness(options.strictness); - const mappedViolations = violations.map((item) => ({ ...item, criterionName: item.description })); + const mappedViolations = violations.map((item) => ({ + ...item, + ...(item.description !== undefined ? { criterionName: item.description } : {}), + })); // Density Calculation: Violations per 100 words const density = (mappedViolations.length / wordCount) * 100; From e319bba09fb1e07af044a55161da89ebb6510313 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:40:19 +0100 Subject: [PATCH 02/24] Exclude docs/research from eslint - Add docs/research/** to eslint ignores; the directory holds local, untracked throwaway research scripts that are not part of the shipped package and should not be type-checked by lint --- eslint.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 8549cd13..4bec57ff 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -10,7 +10,7 @@ import unicorn from "eslint-plugin-unicorn"; export default defineConfig([ // Ignored + housekeeping - { ignores: ["node_modules", "coverage", "dist", "build"] }, + { ignores: ["node_modules", "coverage", "dist", "build", "docs/research/**"] }, { linterOptions: { reportUnusedDisableDirectives: true } }, // Make resolver settings global From 20e0acc8204f1b67858ab7b31bf3f5b023c2799a Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:25 +0100 Subject: [PATCH 03/24] Add vitest.config.ts for stable test resolution - Project had no vitest config; vitest ran on pure defaults - Pin test discovery to tests/**/*.test.ts and inline ora, @langfuse/otel, and @opentelemetry/sdk-node so suites that transitively import agent/observability modules resolve reliably - No change to the 45 suites / 323 tests baseline --- vitest.config.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 vitest.config.ts diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 00000000..b5f7a7b4 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // Tests live under tests/ and follow the *.test.ts convention. + include: ['tests/**/*.test.ts'], + // Inline heavy/ESM-only deps so suites that transitively import the + // agent (ora) or observability (@langfuse/otel, @opentelemetry/sdk-node) + // modules resolve reliably without relying on scattered per-file mocks. + server: { + deps: { + inline: ['ora', '@langfuse/otel', '@opentelemetry/sdk-node'], + }, + }, + }, +}); From 3580e4971c2e6cfa8ce2308d350803c58d43ddba Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:41:59 +0100 Subject: [PATCH 04/24] Add npm run verify aggregate gate - Runs typecheck, lint, and test:run in sequence - Single command for CI and downstream refactor phases to prove the verification baseline --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d9e16b86..3a73ad4d 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint:fix": "eslint . --fix", "test": "vitest", "test:run": "vitest run", - "test:ci": "vitest run --coverage" + "test:ci": "vitest run --coverage", + "verify": "npm run typecheck && npm run lint && npm run test:run" }, "bin": { "vectorlint": "./dist/index.js", From 0fe8700b2e4d2ded0f51ef10e30e1f7fb4f39af5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:47:58 +0100 Subject: [PATCH 05/24] Add typecheck CI job and align test workflow Node version - New typecheck.yml runs tsc --noEmit on push/PR for main and release-docs - Bump test.yml from Node 18 to 20 to satisfy package.json engines>=20.6 - Catches type regressions in CI before merge, not just via ESLint --- .github/workflows/test.yml | 2 +- .github/workflows/typecheck.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/typecheck.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dcf03d6f..06411499 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'npm' - name: Install dependencies diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 00000000..73dad374 --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,28 @@ +name: Typecheck + +on: + push: + branches: [main, release-docs] + pull_request: + branches: [main, release-docs] + +jobs: + typecheck: + name: tsc --noEmit + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run typecheck + run: npm run typecheck From 6c271f121b80e05e5e3d7ae2d7b6c02352755abd Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Mon, 13 Jul 2026 23:58:45 +0100 Subject: [PATCH 06/24] Deprecate --mode agent and fall back to standard - README Agent Mode section replaced with under-review notice linking the audit - --mode agent now warns through the injected logger and runs standard evaluation; the agent executor code is retained (unreachable from the CLI) pending Phase 4 removal - Add logger to EvaluationOptions so the deprecation warning routes through the logging abstraction instead of console - Rewrite orchestrator-agent-output tests to assert deprecation + fallback BREAKING: --mode agent no longer runs the autonomous workspace-agent loop --- README.md | 29 +- src/cli/commands.ts | 3 +- src/cli/orchestrator.ts | 9 +- src/cli/types.ts | 2 + tests/orchestrator-agent-output.test.ts | 1153 ++--------------------- 5 files changed, 80 insertions(+), 1116 deletions(-) diff --git a/README.md b/README.md index 859ea51a..f6511e6a 100644 --- a/README.md +++ b/README.md @@ -145,29 +145,16 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode +## Agent Mode (under review) -Agent mode is for reviews that need context from several files such as -documentation drift and cross-file accuracy. - -Run VectorLint in autonomous agent mode: - -```bash -vectorlint doc.md --mode agent -``` - -For machine-parseable output: - -```bash -vectorlint doc.md --mode agent --output json -``` - -To suppress interactive progress in line output: - -```bash -vectorlint doc.md --mode agent --print -``` +The `--mode agent` flag is under active rework. It currently enables an +autonomous cross-file review mode that is being **removed** in favor of a +bounded harness model. See +[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) +for the decision and the refactor plan. +Until the refactor lands, `--mode agent` prints a deprecation warning and +falls back to standard mode. Do not build integrations against it. ## Contributing diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 369eda1f..50dbffc8 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -82,7 +82,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default) or agent', DEFAULT_REVIEW_MODE) + .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) .option('-p, --print', 'Suppress interactive progress output in agent mode') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') @@ -217,6 +217,7 @@ export function registerMainCommand(program: Command): void { ...(searchProvider ? { searchProvider } : {}), concurrency: config.concurrency, verbose: cliOptions.verbose, + logger: runtimeLogger, debugJson: cliOptions.debugJson, outputFormat: cliOptions.output, mode: cliOptions.mode, diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 6ae4acd4..f8df367a 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1168,6 +1168,9 @@ async function buildAgentRuleScores( return results; } +// Retained in quarantine: unreachable from the CLI after --mode agent was +// deprecated (now falls back to standard evaluation). Removed in Phase 4. +// eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine async function evaluateFilesInAgentMode( targets: string[], options: EvaluationOptions, @@ -1322,7 +1325,11 @@ export async function evaluateFiles( } if (mode === AGENT_REVIEW_MODE) { - return evaluateFilesInAgentMode(targets, options, outputFormat, jsonFormatter); + options.logger?.warn( + '--mode agent is deprecated and now falls back to standard mode. ' + + 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', + ); + // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } for (const file of targets) { diff --git a/src/cli/types.ts b/src/cli/types.ts index a73f55e6..7f95192a 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -10,6 +10,7 @@ import { RdJsonFormatter } from '../output/rdjson-formatter'; import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; +import type { Logger } from '../logging/logger'; export enum OutputFormat { Line = "line", @@ -48,6 +49,7 @@ export interface EvaluationOptions { agentMaxRetries?: number; pricing?: PricingConfig; userInstructionContent?: string; + logger?: Logger; } export interface EvaluationResult { diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index e08bbdaa..0b61ea63 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -2,28 +2,21 @@ import { mkdtempSync, rmSync, writeFileSync } from 'fs'; import * as path from 'path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { evaluateFiles } from '../src/cli/orchestrator'; -import { AGENT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; +import { AGENT_REVIEW_MODE, DEFAULT_REVIEW_MODE, OutputFormat } from '../src/cli/types'; import type { PromptFile } from '../src/prompts/prompt-loader'; import type { LLMProvider } from '../src/providers/llm-provider'; +import type { Logger } from '../src/logging/logger'; import { Severity } from '../src/evaluators/types'; -function makePrompt(params?: { - id?: string; - name?: string; - source?: string; - body?: string; -}): PromptFile { - const id = params?.id ?? 'consistency'; - const name = params?.name ?? 'Consistency'; - const source = params?.source ?? 'packs/default/consistency.md'; - const body = params?.body ?? 'Find inconsistent wording'; - +function makePrompt(): PromptFile { + const id = 'consistency'; + const name = 'Consistency'; return { id, filename: `${id}.md`, - fullPath: source, + fullPath: 'packs/default/consistency.md', pack: 'Default', - body, + body: 'Find inconsistent wording', meta: { id: name, name, @@ -33,91 +26,39 @@ function makePrompt(params?: { }; } -function makeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; -} +type LoggerSpy = Logger & { warn: ReturnType }; -function makeTopLevelOnlyProvider(): LLMProvider { +function makeLogger(): LoggerSpy { return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Top-level without references', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as unknown as LoggerSpy; } -function makeCrossFileTopLevelProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Cross-file issue found', - references: [{ file: 'other.md', startLine: 1, endLine: 1 }], - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; +interface StandardProviderSpies { + provider: LLMProvider; + runPromptStructured: ReturnType; + runAgentToolLoop: ReturnType; } -function makeNoFinalizeProvider(): LLMProvider { - return { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.report_finding.execute({ - kind: 'top-level', - ruleSource: 'packs/default/consistency.md', - message: 'Finding recorded before session ended', - references: [{ file: 'doc.md', startLine: 1, endLine: 1 }], - }); - // intentionally omit finalize_review to simulate missing-finalize scenario - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; +function makeStandardProvider(): StandardProviderSpies { + const runPromptStructured = vi.fn().mockResolvedValue({ + data: { reasoning: 'ok', violations: [] }, + }); + const runAgentToolLoop = vi.fn().mockResolvedValue({ + usage: { inputTokens: 0, outputTokens: 0 }, + }); + const provider = { runPromptStructured, runAgentToolLoop } as unknown as LLMProvider; + return { provider, runPromptStructured, runAgentToolLoop }; } -describe('agent orchestrator output', () => { - const originalIsTTY = Object.getOwnPropertyDescriptor(process.stderr, 'isTTY'); +// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no +// longer reaches the autonomous agent executor: it warns through the injected +// logger and falls back to standard evaluation. The retained agent executor +// code is covered directly by tests/agent/* and removed in Phase 4. +describe('agent mode deprecation', () => { const tempRepos: string[] = []; function createTempRepo(): string { @@ -137,55 +78,15 @@ describe('agent orchestrator output', () => { for (const repo of tempRepos.splice(0, tempRepos.length)) { rmSync(repo, { recursive: true, force: true }); } - if (originalIsTTY) { - Object.defineProperty(process.stderr, 'isTTY', originalIsTTY); - } }); - it('produces agent-mode findings and summary when mode is set to agent', async () => { + it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalFiles).toBe(1); - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.hadOperationalErrors).toBe(false); - }); - - it('includes nested lint usage in the final agent-mode token totals', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { reasoning: 'ok', violations: [] }, - usage: { inputTokens: 7, outputTokens: 3 }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 11, outputTokens: 5 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -194,338 +95,31 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.tokenUsage).toEqual({ - totalInputTokens: 18, - totalOutputTokens: 8, + logger, }); - }); - - it('keeps json output shape consistent with formatter-based structure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const payload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record; - summary?: { files?: number; errors?: number; warnings?: number }; - metadata?: { version?: string; timestamp?: string }; - }; - - expect(payload.files).toBeDefined(); - expect(payload.summary).toBeDefined(); - expect(typeof payload.summary?.files).toBe('number'); - expect(typeof payload.summary?.errors).toBe('number'); - expect(typeof payload.summary?.warnings).toBe('number'); - expect(payload.metadata?.timestamp).toBeTruthy(); - }); - - it('keeps top-level json keys consistent between standard and agent modes', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const standardPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const agentPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record; - - expect(Object.keys(agentPayload).sort()).toEqual( - Object.keys(standardPayload).sort() + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('deprecated'), ); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), + ); + // Standard evaluation ran; the autonomous executor did not. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); - it('emits configured agent progress messages in line output mode', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain(' └ Found no issues in doc.md'); - expect(stderrOutput).not.toContain('[vectorlint]'); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders visible tool invocations and results while hiding internal agent tools', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.search_files.execute({ pattern: '**/*.md' }); - await tools.read_file.execute({ path: 'doc.md' }); - await tools.list_directory.execute({ path: '.' }); - await tools.search_content.execute({ pattern: 'bad phrase', path: '.', glob: '**/*.md' }); - await tools.finalize_review.execute({}); - - return { usage: { inputTokens: 6, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Lint("Find inconsistent wording...")'); - expect(stderrOutput).toContain('Found no issues in doc.md'); - expect(stderrOutput).toContain('Read(doc.md)'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('List(.)'); - expect(stderrOutput).toContain('Listed 1 entry in .'); - expect(stderrOutput).not.toContain('SearchFiles('); - expect(stderrOutput).not.toContain('SearchContent('); - expect(stderrOutput).not.toContain('Finalize('); - }); - - it('renders interactive tool lines without a trailing newline', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const firstToolLine = stderrSpy.mock.calls - .map((call) => String(call[0])) - .find((chunk) => chunk.includes(' └ ')); - - expect(firstToolLine).toBeDefined(); - expect(firstToolLine?.endsWith('\n')).toBe(false); - }); - - it('uses in-place progress updates for repeated tool calls instead of appending only plain lines', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/wordiness.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - makePrompt({ id: 'wordiness', name: 'Wordiness', source: 'packs/default/wordiness.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('\x1b[1A'); - }); - - it('updates progress rule labels as the active lint rule changes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/ai-pattern.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [ - makePrompt({ id: 'ai-pattern', name: 'AI Pattern', source: 'packs/default/ai-pattern.md' }), - makePrompt({ id: 'consistency', name: 'Consistency', source: 'packs/default/consistency.md' }), - ], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('for AI Pattern'); - expect(stderrOutput).toContain('for Consistency'); - }); - - it('shows visible tool retry status after a visible tool failure', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('does not invoke the agent executor in line output mode', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); - const retriable = path.join(repo, 'retriable.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: 'retriable.md' })).rejects.toThrow(); - writeFileSync(retriable, 'hello\n', 'utf8'); - await tools.read_file.execute({ path: 'retriable.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; + const { provider, runPromptStructured, runAgentToolLoop } = makeStandardProvider(); + const logger = makeLogger(); await evaluateFiles([file], { prompts: [makePrompt()], @@ -534,115 +128,23 @@ describe('agent orchestrator output', () => { concurrency: 1, verbose: false, outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, + mode: AGENT_REVIEW_MODE, printMode: false, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading retriable.md'); - expect(stderrOutput).toContain('Retrying Read(retriable.md)...'); - expect(stderrOutput).toContain('Read 1 line from retriable.md'); - }); - - it('shows visible-tool path errors even when path validation fails before file access', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, + logger, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await expect(tools.read_file.execute({ path: '../outside.md' })).rejects.toThrow(); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 2, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Error reading ../outside.md'); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(runPromptStructured).toHaveBeenCalled(); }); - it('shows quality scores in agent line output and keeps operational failures explicit when finalize_review is missing', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - + it('stays silent in standard mode without a logger', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Bad phrase used', - analysis: 'This wording is inconsistent.', - message: 'Use consistent wording', - suggestion: 'Replace bad phrase', - fix: 'better phrase', - rule_quote: 'Avoid vague wording', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - return { usage: { inputTokens: 6, outputTokens: 3 } }; - }, - } as unknown as LLMProvider; + const { provider, runAgentToolLoop } = makeStandardProvider(); const result = await evaluateFiles([file], { prompts: [makePrompt()], @@ -650,549 +152,14 @@ describe('agent orchestrator output', () => { provider, concurrency: 1, verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(0); - expect(stdout).toContain('Use consistent wording'); - expect(stdout).toContain('Quality Scores:'); - expect(stderrSpy).toHaveBeenCalled(); - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Review failed after'); - expect(stderrOutput).not.toContain('Completed review in'); - }); - - it('suppresses progress output when print mode is enabled', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('returns machine-parseable json output without progress text contamination', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stdout).not.toContain('Completed review.'); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('appends a new two-line block when agent work moves to the next file', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const fileOne = path.join(repo, 'doc.md'); - const fileTwo = path.join(repo, 'doc2.md'); - writeFileSync(fileOne, 'bad phrase\n', 'utf8'); - writeFileSync(fileTwo, 'another bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.read_file.execute({ path: 'doc.md' }); - await tools.lint.execute({ file: 'doc.md', ruleSource: 'packs/default/consistency.md' }); - await tools.read_file.execute({ path: 'doc2.md' }); - await tools.lint.execute({ file: 'doc2.md', ruleSource: 'packs/default/consistency.md' }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 4, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([fileOne, fileTwo], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stderrOutput = stderrSpy.mock.calls.map((call) => String(call[0])).join(''); - expect(stderrOutput).toContain('Reviewing doc.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc.md'); - expect(stderrOutput).toContain('Reviewing doc2.md for Consistency'); - expect(stderrOutput).toContain('Read 1 line from doc2.md'); - const firstFileIndex = stderrOutput.indexOf('Reviewing doc.md for Consistency'); - const secondFileIndex = stderrOutput.indexOf('Reviewing doc2.md for Consistency'); - expect(firstFileIndex).toBeGreaterThan(-1); - expect(secondFileIndex).toBeGreaterThan(firstFileIndex); - expect(stderrOutput).toMatch(/Completed review in \d+s\./); - }); - - it('renders top-level findings without explicit references at 1:1 in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeTopLevelOnlyProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('Top-level without references'); - expect(stdout).toContain('1:1'); - }); - - it('prints lazy file headers for non-target referenced findings in line output', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - const otherFile = path.join(repo, 'other.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - writeFileSync(otherFile, 'other content\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeCrossFileTopLevelProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('other.md'); - expect(stdout).toContain('Cross-file issue found'); - }); - - it('scores against every matched file, including clean files without findings', async () => { - const repo = createTempRepo(); - const firstFile = path.join(repo, 'doc.md'); - const secondFile = path.join(repo, 'other.md'); - writeFileSync(firstFile, 'one two three four five six seven eight nine ten\n', 'utf8'); - writeFileSync(secondFile, 'alpha beta gamma delta epsilon zeta eta theta iota kappa\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - reasoning: 'detected issue', - violations: [ - { - line: 1, - quoted_text: 'one', - context_before: '', - context_after: '', - description: 'Issue found', - analysis: 'Needs cleanup.', - message: 'Fix the wording', - suggestion: 'Use clearer wording', - fix: 'clear wording', - rule_quote: 'Be precise', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 3, outputTokens: 2 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([firstFile, secondFile], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Line, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi - .mocked(console.log) - .mock - .calls - .map((call) => String(call[0])) - .join('\n'); - - expect(stdout).toContain('5.0/10'); - }); - - it('keeps canonical rule identity and warning severity aligned across json and rdjson outputs', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, + mode: DEFAULT_REVIEW_MODE, printMode: true, scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const jsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files?: Record }>; - }; - - const jsonIssue = - jsonPayload.files?.['doc.md']?.issues?.find( - (issue) => issue.rule === 'Default.Consistency' - ) ?? jsonPayload.files?.['doc.md']?.issues?.[0]; - - vi.mocked(console.log).mockClear(); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const rdjsonPayload = JSON.parse( - String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - diagnostics?: Array<{ code?: { value?: string }; severity?: string }>; - }; - - const rdjsonDiagnostic = - rdjsonPayload.diagnostics?.find( - (diagnostic) => diagnostic.code?.value === 'Default.Consistency' - ) ?? rdjsonPayload.diagnostics?.[0]; - - expect(jsonIssue?.rule).toBe('Default.Consistency'); - expect(jsonIssue?.severity).toBe(Severity.WARNING); - expect(rdjsonDiagnostic?.code?.value).toBe('Default.Consistency'); - expect(rdjsonDiagnostic?.severity).toBe(Severity.WARNING); - }); - - it('keeps rdjson output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.RdJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('keeps vale-json output machine-parseable without interactive progress writes', async () => { - Object.defineProperty(process.stderr, 'isTTY', { - configurable: true, - value: true, - }); - const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); - - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.ValeJson, - mode: AGENT_REVIEW_MODE as never, - printMode: false, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - const stdout = vi.mocked(console.log).mock.calls.map((call) => String(call[0])).join('\n'); - expect(stdout).not.toContain('Reviewing'); - expect(stdout).not.toContain(' └ '); - expect(stderrSpy).not.toHaveBeenCalled(); - }); - - it('surfaces findings recorded before missing finalize while still reporting an operational failure', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider: makeNoFinalizeProvider(), - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.totalWarnings).toBeGreaterThan(0); - expect(result.requestFailures).toBe(0); - expect(result.hadOperationalErrors).toBe(true); - }); - - it('passes userInstructionContent into the agent system prompt', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let capturedSystemPrompt = ''; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - const systemPrompt = params.systemPrompt; - capturedSystemPrompt = typeof systemPrompt === 'string' ? systemPrompt : ''; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - userInstructionContent: 'Always enforce concise phrasing.', - } as never); - - expect(capturedSystemPrompt).toContain('User Instructions (from VECTORLINT.md):'); - expect(capturedSystemPrompt).toContain('Always enforce concise phrasing.'); - }); - - it('counts provider tool-loop failures as request failures in agent mode', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop() { - return Promise.reject(new Error('provider request failed')); - }, - } as unknown as LLMProvider; - - const result = await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(result.hadOperationalErrors).toBe(true); - expect(result.requestFailures).toBe(1); - }); - - it('passes a default agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(10); - }); - - it('passes configured agent retry budget to the provider tool loop', async () => { - const repo = createTempRepo(); - const file = path.join(repo, 'doc.md'); - writeFileSync(file, 'bad phrase\n', 'utf8'); - - let receivedParams: Record | undefined; - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ data: { reasoning: 'ok', violations: [] } }); - }, - runAgentToolLoop: async (params: Record) => { - receivedParams = params; - const tools = params.tools as Record Promise }>; - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - await evaluateFiles([file], { - prompts: [makePrompt()], - rulesPath: undefined, - provider, - concurrency: 1, - verbose: false, - outputFormat: OutputFormat.Json, - mode: AGENT_REVIEW_MODE as never, - printMode: true, - agentMaxRetries: 4, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - } as never); - - expect(receivedParams?.maxRetries).toBe(4); + // No deprecation warning and no executor invocation in standard mode. + expect(runAgentToolLoop).not.toHaveBeenCalled(); + expect(result.totalFiles).toBe(1); }); }); From a2e9f25c8935163285235736ac70ff5926d47b95 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:10:35 +0100 Subject: [PATCH 07/24] Append Phase 1 completion note to harness audit - Record the shifted 2026-07-13 baseline: only tsc --noEmit was failing; lint and test:run were already green, so the audit's lint and four-suite module-resolution failures are stale - Note the durable Phase 1 gates (typecheck, verify, vitest.config, docs/research lint exclusion, Node 20 typecheck/test workflows) - Point to the --mode agent deprecation and standard fallback as the precondition for Phases 2-5; record that no spec or architecture doc is superseded in this appendix --- ...0-vectorlint-harness-architecture-audit.md | 380 ++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md new file mode 100644 index 00000000..51161ce0 --- /dev/null +++ b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md @@ -0,0 +1,380 @@ +# VectorLint Harness Architecture Audit + +Date: 2026-07-10 + +Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. + +## Product Decision + +VectorLint should not keep the current autonomous workspace-agent mode. + +The retained agent-like capability should be a constrained reader executor: a model or headless agent can read the target content in sections when that is useful for context management, but it should not search the workspace, inspect arbitrary files, perform top-level workspace checks, or rewrite the rule being evaluated. + +The intended execution model is: + +- `direct`: send target content and rule in one structured review call. +- `reader`: give the executor a read-section capability over the target content only. +- `auto`: choose `direct` for normal-sized inputs and `reader` for large inputs or explicit context-management needs. + +This is an execution strategy decision, not a return to VectorLint-as-agent. + +## Product Direction Under Review + +VectorLint should become a programmable, bounded, on-page content review harness that another caller can invoke. + +The caller can be Codex, Claude Code, another coding agent, CI, a local CLI wrapper, or a future service. That caller owns exploration, cross-page reasoning, research, and context gathering. VectorLint owns the constrained review of target content against source-backed rules and returns structured findings, scores, diagnostics, and usage metadata. + +In practical terms: + +- VectorLint reviews the target page or explicitly supplied content. +- External context is caller-supplied, not discovered by VectorLint. +- Rules define review behavior, granularity, severity, and scoring constraints. +- Executors can be API models, headless local agents, or constrained reader executors, but they receive the same review request contract. +- VectorLint should not expose a model-controlled workspace tool loop as its core product surface. + +## Objective System Audit Summary + +Separate from product direction, the current codebase has several system-level liabilities: + +- Build health is weak: `npx tsc --noEmit`, `npm run lint`, and parts of `npm run test:run` fail in the current workspace. +- Runtime contracts are duplicated across hand-written TypeScript, JSON-schema objects, Zod schemas, and formatter-specific shapes. +- Standard mode and agent mode project the same underlying findings differently. +- CLI orchestration owns too many responsibilities: config, file matching, evaluation, scoring, output formatting, debug artifacts, and agent routing. +- Evidence verification and counting are inconsistent enough to affect exit behavior. +- Observability and debug paths can record full prompts, content, and outputs. +- Documentation and implementation disagree on the current agent-mode topology and tool contracts. + +## Architecture Verdict + +The current codebase contains two execution models at once: + +1. The older evaluator path: one structured model call per rule or chunk. +2. The newer autonomous workspace-agent path: VectorLint gives a model tools to read/search the workspace and call lint as a nested tool. + +The second path should not continue as-is. The useful piece is not "agent mode" broadly; it is target-aware reading for context management. The current implementation also duplicates result projection, scoring, evidence validation, output formatting, and configuration behavior from the first path. That is why the architecture feels harder to maintain: the code is not just messy; it is serving two different ownership models. + +The highest-leverage change is to define a neutral review-domain contract and make every executor implement that contract. + +## Proposed Core Contract + +The contract should be centered on review, not providers or agents. + +```ts +interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; +} + +interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; +} +``` + +API models, reasoning models, and headless CLI agents become adapter implementations behind this contract. The CLI becomes orchestration around request creation, result formatting, and exit behavior. + +## Major Findings + +### 1. The Current Agent Surface Is Broader Than The Decided Execution Model + +Evidence: + +- `README.md:148` presents `--mode agent` as autonomous cross-file review mode. +- `src/agent/prompt-builder.ts:43` instructs the model to perform workspace-level checks. +- `src/agent/executor.ts:670` exposes `read_file`. +- `src/agent/executor.ts:715` exposes `search_content`. + +Impact: + +VectorLint still behaves like it owns exploration. That puts it in competition with external coding agents instead of becoming a harness those agents can use. It also overshoots the decided reader-executor model, which only needs target-scoped reading for context management. + +Recommendation: + +Remove the current autonomous workspace-agent surface. Preserve only a constrained reader executor if needed: + +- It may read sections of the target content. +- It may not search the workspace. +- It may not read arbitrary files. +- It may not perform top-level workspace findings. +- It may not override source-backed rules. + +### 2. The Provider Interface Mixes Structured Review With Autonomous Tool Loops + +Evidence: + +- `src/providers/llm-provider.ts:28` requires both `runPromptStructured` and `runAgentToolLoop`. +- `src/providers/vercel-ai-provider.ts:136` implements the agent loop on the same provider. +- `src/agent/executor.ts:796` depends on the provider-level agent loop. + +Impact: + +Every future executor has to pretend to be both a structured model provider and an autonomous agent runner. That is the wrong abstraction for headless CLI support and for a constrained reader executor. + +Recommendation: + +Split the contracts: + +- `StructuredModelClient` for API structured output. +- `HeadlessReviewExecutor` for local CLI agents. +- `ReaderReviewExecutor` for target-scoped read-section execution. +- `ReviewExecutor` as the stable domain-level interface. +- `TelemetrySink` or observability decoration as an optional cross-cutting concern. + +### 3. Runtime Type Contracts Are Not Strong Enough For A Harness + +Evidence: + +- `src/providers/vercel-ai-provider.ts:119` returns `output as T`. +- `src/providers/llm-provider.ts:9` defines tool input and output as `unknown`. +- `src/agent/tools-registry.ts:21` exposes tools without typed output contracts. +- `npx tsc --noEmit` currently fails in core files. + +Impact: + +The code presents types as guarantees, but several are only assertions. A programmable harness needs reliable runtime validation because external callers and headless adapters will depend on these contracts. + +Recommendation: + +Tie structured execution to concrete Zod schemas or canonical parsers. Introduce typed tool contracts only if tools remain. Add a `typecheck` script and wire it into CI and build. + +### 4. Standard Mode And Agent Mode Project Results Differently + +Evidence: + +- `src/cli/orchestrator.ts:626` derives standard check severity from scoring. +- `src/agent/executor.ts:431` stamps agent findings with prompt severity. +- `src/agent/executor.ts:598` flattens judge criteria before recording findings. +- `src/cli/orchestrator.ts:1216` builds agent scores only for line output. +- `src/cli/orchestrator.ts:1237` emits structured output without agent scores or diagnostics. + +Impact: + +The same underlying review can produce different severity, rule identity, score data, JSON shape, and exit behavior depending on mode. That makes VectorLint hard to trust as a machine-facing gate. + +Recommendation: + +Extract one shared result projection pipeline: + +```text +PromptEvaluationResult + -> verified findings + -> rule and criterion identity + -> severity and score + -> diagnostics + -> output formatter +``` + +Every executor should return into this pipeline. + +### 5. The On-Page Boundary Is Not Enforced + +Evidence: + +- `src/agent/executor.ts:585` lets `lint` resolve any file inside `workspaceRoot`. +- `src/agent/executor.ts:670` lets `read_file` read any workspace file. +- `src/agent/executor.ts:715` lets `search_content` scan workspace content. +- `src/agent/executor.ts:134` allows model-supplied `reviewInstruction`. +- `src/agent/executor.ts:583` replaces the source-backed prompt body with the model-supplied override. + +Impact: + +Prompt-injected page content can indirectly steer the model to read non-target files or rewrite the rule being evaluated. That breaks deterministic review semantics and creates privacy and security risk. + +Recommendation: + +For the new harness, enforce a target/context allowlist: + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller explicitly includes those files as context. +- Reader execution can page through target content only. +- Rule bodies are source-backed and caller-authored, not model-authored. + +### 6. Evidence Handling Can Misreport Findings + +Evidence: + +- Standard mode skips unverifiable quotes but can still count original surfaced violation totals in some paths. +- `src/agent/executor.ts:415` attempts to locate evidence. +- `src/agent/executor.ts:426` falls back to the model-provided line when location fails. + +Impact: + +Standard mode can fail a run without printing the issue that caused the failure. Agent mode can surface hallucinated evidence as if it were verified. + +Recommendation: + +Use one evidence verifier. Count only emitted, verified findings. If evidence cannot be located, return a diagnostic and either skip the finding or mark the run operationally failed. + +### 7. Cost And Work Are Not Bounded Enough + +Evidence: + +- `src/providers/vercel-ai-provider.ts:161` defaults agent loops to 1000 steps. +- `src/agent/executor.ts:577` allows repeated nested `lint` calls. +- `src/evaluators/base-evaluator.ts:213` runs one model call per chunk. +- `src/agent/executor.ts:680` and `src/agent/executor.ts:715` glob before applying result caps. + +Impact: + +A single review can multiply cost through agent steps, nested lint calls, rules, chunks, and workspace search. Even a constrained reader executor will be slower than a direct call, so `auto` must have clear thresholds and budgets. + +Recommendation: + +Add explicit budgets: + +- Max target bytes. +- Max caller context bytes. +- Max chunks per rule. +- Max model calls per review. +- Max findings per rule. +- Max wall-clock duration. +- Max headless executor retries. + +### 8. Documentation Still Points At The Old Product + +Evidence: + +- `README.md:148` documents autonomous workspace-agent mode. +- `docs/specs/2026-03-17-agentic-capabilities-design.md:24` frames the roadmap around cross-document agent mode. +- The same spec describes implementation details that are not true now: one agent per rule, final `AgentFindingSchema`, AbortSignal propagation, ripgrep JSON, and Vale JSON omission of agent findings. + +Impact: + +Future contributors will optimize toward the wrong architecture and trust contracts that the code does not implement. + +Recommendation: + +Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and migration plan. + +### 9. Secrets And Sensitive Content Need Better Handling + +Evidence: + +- `src/config/global-config.ts:13` says the global config stores API keys. +- `src/config/global-config.ts:76` creates the config directory with default permissions. +- `src/config/global-config.ts:81` writes the config file with default permissions. +- `src/observability/langfuse-observability.ts:66` records inputs. +- `src/observability/langfuse-observability.ts:67` records outputs. +- `src/providers/vercel-ai-provider.ts:59` can log full prompts and content. + +Impact: + +Provider keys, unreleased docs, caller context, and model outputs can leak through filesystem permissions, telemetry, debug logs, or persisted artifacts. + +Recommendation: + +Use private file modes for config and review artifacts. Make payload telemetry a separate opt-in from metadata telemetry. Add redaction or safe debug modes. + +## Verification Results + +Commands run from `/Users/klinsmann/Projects/TinyRocketLabs/vectorlint`: + +```bash +npm run lint +npm run test:run +npx tsc --noEmit +``` + +Results: + +- `npm run lint` failed before linting because typed ESLint rules were applied to a `.cjs` research script without parser type information. +- `npm run test:run` passed most tests but failed 4 suites during module resolution for `ora` and `@langfuse/otel`. +- `npx tsc --noEmit` failed with contract errors in agent, orchestrator, observability, logging, and provider modules. + +## Recommended Refactor Sequence + +### Phase 1: Stop The Bleeding + +1. Remove or hide the current autonomous workspace-agent mode. +2. Fix `tsc --noEmit`. +3. Fix test module resolution. +4. Fix lint configuration for non-TypeScript research scripts. +5. Add `typecheck` to CI and build validation. + +### Phase 2: Define The Harness Contract + +1. Create a neutral review-domain module. +2. Define `ReviewRequest`, `ReviewRule`, `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, and `ReviewDiagnostic`. +3. Define target/context boundary rules. +4. Define structured output shape once. +5. Define execution strategy: `direct`, `reader`, and `auto`. + +### Phase 3: Share Result Projection + +1. Extract evidence location and verification. +2. Extract filtering. +3. Extract severity and scoring. +4. Extract output formatter inputs. +5. Make standard mode and any executor mode use the same projection path. + +### Phase 4: Replace The Agent Loop With Executors + +1. Keep API model execution as direct execution. +2. Add target-scoped reader execution as another executor. +3. Add headless CLI execution behind the same review contract if still useful. +4. Pass the same review request to each executor. +5. Remove model-controlled rule overrides. +6. Remove workspace read/search tools from core review. + +### Phase 5: Update Documentation + +1. Mark the agentic capabilities spec superseded. +2. Write a new harness architecture spec. +3. Update README and CLI docs. +4. Document migration from autonomous workspace-agent mode to direct/reader/auto execution. + +## Suggested Target Architecture + +```text +CLI / API / External Agent + -> ReviewRequestBuilder + -> ReviewExecutor + -> ApiModelExecutor + -> ReaderExecutor + -> HeadlessCliExecutor + -> ResultProjection + -> EvidenceVerifier + -> ViolationFilter + -> ScoreCalculator + -> Diagnostics + -> OutputFormatter + -> line + -> json + -> rdjson + -> vale-json +``` + +The important inversion is that VectorLint no longer asks, "What tools should this agent use?" It asks, "Given this target, these rules, this context, this execution strategy, and this budget, what findings can be verified?" + +## Final Recommendation + +The decided direction is cleaner than the current architecture. The current pain is not a sign that VectorLint is too complex; it is a sign that the codebase is carrying two ownership models and several duplicated runtime contracts. + +Make VectorLint the harness. Let external agents be agents. Keep reader execution only as a bounded way to manage target-content context. + +--- + +## Appendix: Phase 1 — Completed (2026-07-13) + +Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on branch `codex/ci/harness-stop-the-bleeding`. It shifts the verification baseline recorded in "Verification Results" above and addresses Finding 1 at the CLI boundary. + +**Shifted baseline.** The 2026-07-13 pre-work capture differed from the original verification results: `npm run lint` and `npm run test:run` already passed, so the earlier lint failure and the four `ora` / `@langfuse/otel` module-resolution failures were stale. Only `npx tsc --noEmit` was still failing. Phase 1 cleared those remaining type errors (narrowed at boundaries; no strict compiler options relaxed) and added durable gates so the baseline cannot silently regress: + +- `npm run typecheck` (`tsc --noEmit`) and `npm run verify` (typecheck + lint + test:run) scripts. +- `vitest.config.ts` for stable test module resolution. +- `docs/research/**` excluded from linting. +- `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. + +**Current state.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the autonomous workspace-agent surface is deprecated at the CLI boundary: `--mode agent` emits a deprecation warning and falls back to standard evaluation. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. The deprecation notice and pointer to this audit live in the README `## Agent Mode` section and the `--mode` help text. + +This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. From 888f0271e30b0e0e1fa5d2773ca89f94cc415d58 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:22:49 +0100 Subject: [PATCH 08/24] Add root .vectorlint.ini for reproducible smoke validation - Commit a repository-native .vectorlint.ini using the bundled VectorLint preset (verbatim `vectorlint init` template). - The required validation commands `npm start -- README.md --output line` and `npm start -- README.md --mode agent` previously failed from the committed branch with "Missing configuration file"; they had relied on an untracked, deleted VECTORLINT.md. - Both smoke commands now run from a clean checkout with no untracked setup files; --mode agent still warns and falls back to standard mode. Refs: .agent-runs/2026-07-13-223741-harness-refactor/reports/01-stop-the-bleeding.md --- .vectorlint.ini | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vectorlint.ini diff --git a/.vectorlint.ini b/.vectorlint.ini new file mode 100644 index 00000000..d5b0d845 --- /dev/null +++ b/.vectorlint.ini @@ -0,0 +1,9 @@ +# VectorLint Configuration +# Global settings +RulesPath= +Concurrency=4 +DefaultSeverity=warning + +# Default rules for all markdown files +[**/*.md] +RunRules=VectorLint From 99f73c22ae728cb3dfb904dd0208b881322ac09b Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 00:56:45 +0100 Subject: [PATCH 09/24] Add neutral review-domain contract module - Introduce src/review/ with typed + Zod-validated ReviewRequest/Result contracts, boundary helpers, budget defaults/enforcement, model-call selection, ReviewExecutor interface, and a PromptFile request builder - Every external shape has a paired strict Zod schema; legacy scoring-mode, rubric, and model-authored rule-override fields are rejected - modelCall is single | agent | auto; chooseModelCall() resolves auto - On-page boundary (target + caller context only) via buildScope/isInScope with pure lexical URI normalization; no filesystem reads - BudgetExceededError extends the repository VectorlintError base - Purely additive: no changes outside src/review/ and tests/review/ - tests/review/ covers surface, schemas, budget, boundary, results, model-call selection, and the request builder (46 tests) --- src/review/README.md | 42 ++++++ src/review/boundary.ts | 67 ++++++++++ src/review/budget.ts | 83 ++++++++++++ src/review/executor.ts | 53 ++++++++ src/review/index.ts | 15 +++ src/review/request-builder.ts | 88 ++++++++++++ src/review/schemas.ts | 135 +++++++++++++++++++ src/review/types.ts | 192 +++++++++++++++++++++++++++ tests/review/boundary.test.ts | 39 ++++++ tests/review/budget.test.ts | 67 ++++++++++ tests/review/executor.test.ts | 46 +++++++ tests/review/module-surface.test.ts | 34 +++++ tests/review/request-builder.test.ts | 97 ++++++++++++++ tests/review/result.test.ts | 95 +++++++++++++ tests/review/types.test.ts | 74 +++++++++++ 15 files changed, 1127 insertions(+) create mode 100644 src/review/README.md create mode 100644 src/review/boundary.ts create mode 100644 src/review/budget.ts create mode 100644 src/review/executor.ts create mode 100644 src/review/index.ts create mode 100644 src/review/request-builder.ts create mode 100644 src/review/schemas.ts create mode 100644 src/review/types.ts create mode 100644 tests/review/boundary.test.ts create mode 100644 tests/review/budget.test.ts create mode 100644 tests/review/executor.test.ts create mode 100644 tests/review/module-surface.test.ts create mode 100644 tests/review/request-builder.test.ts create mode 100644 tests/review/result.test.ts create mode 100644 tests/review/types.test.ts diff --git a/src/review/README.md b/src/review/README.md new file mode 100644 index 00000000..bc2a120b --- /dev/null +++ b/src/review/README.md @@ -0,0 +1,42 @@ +# `src/review/` — Review-Domain Contract + +The neutral, implementation-neutral contract every executor programs against. +A caller builds a `ReviewRequest`; an executor returns a `ReviewResult`. + +## What lives here + +- `types.ts` — all review-domain interfaces (`ReviewTarget`, `ReviewRule`, + `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, + `ReviewDiagnostic`, `ReviewResult`, `ReviewRequest`, `ReviewModelCall`). +- `schemas.ts` — Zod schemas mirroring every external shape (boundary + validation). All schemas are strict; legacy scoring-mode, rubric, and + model-authored rule-override fields are rejected. +- `budget.ts` — `DEFAULT_REVIEW_BUDGET`, `REVIEW_BUDGET_SCHEMA`, + `enforceBudget()`, and `BudgetExceededError` (extends `VectorlintError`). +- `boundary.ts` — `buildScope()` / `isInScope()` enforcing the on-page boundary. +- `executor.ts` — `ReviewExecutor` interface, `REVIEW_MODEL_CALLS` + (`single | agent | auto`), `chooseModelCall()`. +- `request-builder.ts` — `buildReviewRequest()` bridging `PromptFile` to + `ReviewRequest`. + +## On-page boundary (audit Finding #5) + +- Target content is always in scope. +- Caller-supplied context is in scope. +- Workspace reads are out of scope unless the caller includes them as context. +- Agent model calls page through target content only. +- Rule bodies are source-backed and caller-authored, never model-authored. + +## Execution strategy + +`modelCall: single | agent | auto` selects how the reviewer model is invoked, +not how rules are scored. `single` sends target + rule in one structured call; +`agent` gives the executor a target-scoped read-section capability; `auto` +picks `single` for normal-sized inputs and `agent` for large ones. + +## Wiring status + +Phase 2 (this module) is purely additive and is **not** wired into the CLI. +Phase 3 emits `ReviewResult` via shared finding processing; Phase 4 implements +executors behind `ReviewExecutor` and removes the old agent loop. See +`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`. diff --git a/src/review/boundary.ts b/src/review/boundary.ts new file mode 100644 index 00000000..8dc5bb0c --- /dev/null +++ b/src/review/boundary.ts @@ -0,0 +1,67 @@ +import type { ReviewScope } from './types'; + +const FILE_URI_RE = /^file:\/\/(?[^/]*)(?\/.*)?$/; + +/** + * Lexically resolves '.' and '..' segments in a path string. Pure: performs no + * filesystem reads. The normalization philosophy is adapted from + * src/agent/path-utils.ts (resolveWithinRoot), which is removed in Phase 4; + * copied here so src/review/ has no agent import and no filesystem access. + * + * Symlink canonicalization is intentionally NOT performed inside the contract: + * callers are responsible for canonicalizing real files into target/context + * URIs before building a ReviewRequest. + */ +function resolveDotSegments(input: string): string { + const segments: string[] = []; + for (const segment of input.split('/')) { + if (segment === '' || segment === '.') continue; + if (segment === '..') { + segments.pop(); + continue; + } + segments.push(segment); + } + return (input.startsWith('/') ? '/' : '') + segments.join('/'); +} + +/** + * Normalizes a review URI by resolving '.'/'..' segments in its path. file:// + * URIs are normalized structurally; other (virtual, in-memory) URIs are + * returned as-is after dot resolution of their path-like suffix. + */ +export function normalizeReviewUri(uri: string): string { + const match = FILE_URI_RE.exec(uri); + if (match?.groups) { + const authority = match.groups['authority'] ?? ''; + const rawPath = match.groups['path'] ?? '/'; + return `file://${authority}${resolveDotSegments(rawPath)}`; + } + return uri; +} + +/** + * Builds the on-page boundary scope (audit Finding #5) from the target URI and + * any caller-supplied context URIs. Only these URIs are in scope; arbitrary + * workspace files are out of scope unless the caller explicitly includes them. + */ +export function buildScope(params: { + targetUri: string; + contextUris?: readonly string[]; +}): ReviewScope { + const allowedUris = new Set(); + allowedUris.add(normalizeReviewUri(params.targetUri)); + for (const uri of params.contextUris ?? []) { + allowedUris.add(normalizeReviewUri(uri)); + } + return { allowedUris }; +} + +/** + * Returns true iff `uri` (after normalization) is the target or caller-supplied + * context. Path-traversal segments are resolved before comparison, so a + * traversal-style path cannot masquerade as an in-scope URI. + */ +export function isInScope(scope: ReviewScope, uri: string): boolean { + return scope.allowedUris.has(normalizeReviewUri(uri)); +} diff --git a/src/review/budget.ts b/src/review/budget.ts new file mode 100644 index 00000000..e7d4cd8b --- /dev/null +++ b/src/review/budget.ts @@ -0,0 +1,83 @@ +import { z } from 'zod'; +import type { ReviewBudget } from './types'; +import { VectorlintError } from '../errors'; + +/** + * Sensible default bounds for a single review (audit Finding #7). Frozen so a + * shared default object cannot be accidentally mutated by a caller. + * + * Note: there is intentionally no finding cap (`maxFindingsPerRule`) — + * VectorLint reports every verified issue it finds — and no headless retry + * budget (`maxHeadlessRetries`) because the decided architecture has no + * headless executor. + */ +export const DEFAULT_REVIEW_BUDGET: Readonly = Object.freeze({ + maxTargetBytes: 1_000_000, + maxCallerContextBytes: 500_000, + maxChunksPerRule: 20, + maxModelCallsPerReview: 50, + maxWallClockMs: 5 * 60_000, +}); + +/** Zod schema mirroring {@link ReviewBudget}, applying defaults for omitted fields. */ +export const REVIEW_BUDGET_SCHEMA = z + .object({ + maxTargetBytes: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxTargetBytes), + maxCallerContextBytes: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxCallerContextBytes), + maxChunksPerRule: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxChunksPerRule), + maxModelCallsPerReview: z + .number() + .int() + .positive() + .default(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview), + maxWallClockMs: z.number().int().positive().default(DEFAULT_REVIEW_BUDGET.maxWallClockMs), + }) + .strict(); + +/** Current resource usage, checked against a {@link ReviewBudget}. */ +export interface BudgetUsage { + modelCalls: number; + elapsedMs: number; +} + +/** + * Thrown by {@link enforceBudget} when current usage violates a hard limit. + * Extends the repository's canonical error base (VectorlintError). + */ +export class BudgetExceededError extends VectorlintError { + constructor( + message: string, + public readonly limit: keyof ReviewBudget, + public readonly actual: number, + ) { + super(message, 'BUDGET_EXCEEDED'); + this.name = 'BudgetExceededError'; + } +} + +/** + * Throws {@link BudgetExceededError} if current usage violates a hard limit. + * Executors call this before/after each model call and on timeout checks. + * Only runtime counters (model calls, wall clock) are enforced here; size + * limits (target/context bytes, chunks) are enforced at request-build time. + */ +export function enforceBudget(budget: ReviewBudget, usage: BudgetUsage): void { + if (usage.modelCalls > budget.maxModelCallsPerReview) { + throw new BudgetExceededError( + `model calls (${usage.modelCalls}) exceed maxModelCallsPerReview (${budget.maxModelCallsPerReview})`, + 'maxModelCallsPerReview', + usage.modelCalls, + ); + } + if (usage.elapsedMs > budget.maxWallClockMs) { + throw new BudgetExceededError( + `elapsed time (${usage.elapsedMs}ms) exceeds maxWallClockMs (${budget.maxWallClockMs}ms)`, + 'maxWallClockMs', + usage.elapsedMs, + ); + } +} diff --git a/src/review/executor.ts b/src/review/executor.ts new file mode 100644 index 00000000..2eef9011 --- /dev/null +++ b/src/review/executor.ts @@ -0,0 +1,53 @@ +import type { ReviewModelCall, ReviewRequest, ReviewResult } from './types'; + +/** + * Above this target byte size (roughly the existing chunking threshold in + * bytes), `auto` model-call selection prefers the agent executor so the + * reviewer can page through target content for context management. + */ +export const AGENT_MODEL_CALL_BYTE_THRESHOLD = 600_000; + +/** + * The single source of truth for reviewer model-call strategies. Kept in sync + * with the {@link ReviewModelCall} union via `satisfies`. + */ +export const REVIEW_MODEL_CALLS = ['single', 'agent', 'auto'] as const satisfies readonly ReviewModelCall[]; + +/** + * The stable domain-level interface every executor implements. Single and + * agent executors are implementations behind this contract (audit Finding #2). + * A headless/autonomous workspace executor is explicitly NOT part of this + * contract. + */ +export interface ReviewExecutor { + run(request: ReviewRequest): Promise; +} + +/** + * Agent-call capability: a bounded way to page through target content for + * context management. NOT a workspace tool (audit Product Decision). An agent + * executor receives this capability scoped to the target only. + */ +export interface ReviewTargetReadCapability { + /** Read a 1-based [startLine, endLine] window of the target content. */ + readTargetSection( + startLine: number, + endLine: number, + ): Promise<{ startLine: number; endLine: number; content: string }>; +} + +/** + * Resolves 'auto' to 'single' or 'agent'. Single for normal-sized inputs; + * agent for large inputs or multi-rule runs that benefit from paging. + */ +export function chooseModelCall( + modelCall: ReviewModelCall, + signal: { targetBytes: number; rules: number }, +): 'single' | 'agent' { + if (modelCall === 'single') return 'single'; + if (modelCall === 'agent') return 'agent'; + if (signal.targetBytes > AGENT_MODEL_CALL_BYTE_THRESHOLD || signal.rules > 5) { + return 'agent'; + } + return 'single'; +} diff --git a/src/review/index.ts b/src/review/index.ts new file mode 100644 index 00000000..c44dbe66 --- /dev/null +++ b/src/review/index.ts @@ -0,0 +1,15 @@ +/** + * Neutral review-domain contract for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. The CLI, headless adapters, + * and external callers all build a ReviewRequest and receive a ReviewResult. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + */ +export * from './types'; +export * from './schemas'; +export * from './budget'; +export * from './boundary'; +export * from './executor'; +export * from './request-builder'; diff --git a/src/review/request-builder.ts b/src/review/request-builder.ts new file mode 100644 index 00000000..72500596 --- /dev/null +++ b/src/review/request-builder.ts @@ -0,0 +1,88 @@ +import type { PromptFile } from '../schemas/prompt-schemas'; +import type { + ReviewBudget, + ReviewContext, + ReviewModelCall, + ReviewOutputPolicy, + ReviewRequest, + ReviewRule, + ReviewTarget, +} from './types'; +import { DEFAULT_REVIEW_BUDGET } from './budget'; +import { ValidationError } from '../errors'; + +/** Default output policy: include usage metadata, never record payloads. */ +export const DEFAULT_REVIEW_OUTPUT_POLICY: Readonly = Object.freeze({ + includeUsage: true, + recordPayloadTelemetry: false, +}); + +/** Optional overrides applied by {@link buildReviewRequest}. */ +export interface ReviewRequestBuilderConfig { + budget?: ReviewBudget; + outputPolicy?: ReviewOutputPolicy; + modelCall?: ReviewModelCall; +} + +export interface BuildReviewRequestParams { + target: ReviewTarget; + /** Existing prompt files to convert into source-backed ReviewRules. */ + prompts: readonly PromptFile[]; + /** Caller-supplied, in-scope context, passed through unchanged. */ + context?: ReviewContext[]; + config?: ReviewRequestBuilderConfig; +} + +/** Builds the stable ReviewRule id formatted Pack.Rule (mirrors src/agent/rule-id.ts). */ +function toReviewRuleId(prompt: PromptFile): string { + const pack = prompt.pack || 'Default'; + return `${pack}.${prompt.meta.id}`; +} + +/** + * Converts a single PromptFile into a source-backed ReviewRule, mapping only + * fields that belong in the neutral contract. Legacy `meta.type`, `evaluator`, + * `criteria`, and judge/rubric fields are intentionally NOT copied. + */ +function toReviewRule(prompt: PromptFile): ReviewRule { + const rule: ReviewRule = { + id: toReviewRuleId(prompt), + source: prompt.fullPath, + body: prompt.body, + }; + if (prompt.meta.name !== undefined) { + rule.name = prompt.meta.name; + } + // Severity is a string enum whose values ('error' | 'warning') match + // ReviewSeverity exactly, so it maps cleanly without transformation. + if (prompt.meta.severity !== undefined) { + rule.severity = prompt.meta.severity; + } + return rule; +} + +/** + * Builds a {@link ReviewRequest} from existing {@link PromptFile}s plus a + * target. This is the conservative bridge Phase 4 uses to convert the current + * prompt pipeline into the review contract without changing how prompts load. + * Throws a {@link ValidationError} when no prompts are supplied. + */ +export function buildReviewRequest(params: BuildReviewRequestParams): ReviewRequest { + const { target, prompts, context } = params; + if (prompts.length === 0) { + throw new ValidationError('buildReviewRequest requires at least one prompt.'); + } + + const config = params.config; + const request: ReviewRequest = { + target, + rules: prompts.map(toReviewRule), + budget: config?.budget ?? DEFAULT_REVIEW_BUDGET, + outputPolicy: config?.outputPolicy ?? DEFAULT_REVIEW_OUTPUT_POLICY, + modelCall: config?.modelCall ?? 'auto', + }; + if (context !== undefined) { + request.context = context; + } + return request; +} diff --git a/src/review/schemas.ts b/src/review/schemas.ts new file mode 100644 index 00000000..a1132cd3 --- /dev/null +++ b/src/review/schemas.ts @@ -0,0 +1,135 @@ +import { z } from 'zod'; +import { REVIEW_BUDGET_SCHEMA } from './budget'; + +/** + * Zod schemas mirroring src/review/types.ts (boundary validation). + * + * Every external review-domain shape has a paired schema here so callers and + * external adapters can validate untrusted input at the system boundary + * (audit Finding #3). Schemas are intentionally strict: unknown keys are + * rejected so legacy scoring-mode, rubric, and model-authored rule-override + * fields cannot leak into the new contract. + */ + +export const REVIEW_SEVERITY_SCHEMA = z.enum(['error', 'warning']); + +export const REVIEW_VIOLATION_CONDITION_SCHEMA = z + .object({ + id: z.string().min(1), + description: z.string().min(1), + }) + .strict(); + +export const REVIEW_TARGET_SCHEMA = z + .object({ + uri: z.string().min(1), + content: z.string(), + contentType: z.string().min(1), + byteLength: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RULE_SCHEMA = z + .object({ + id: z.string().min(1), + source: z.string().min(1), + body: z.string(), + name: z.string().optional(), + severity: REVIEW_SEVERITY_SCHEMA.default('warning'), + violationConditions: z.array(REVIEW_VIOLATION_CONDITION_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_CONTEXT_SCHEMA = z + .object({ + label: z.string().min(1), + content: z.string(), + relation: z.string().optional(), + uri: z.string().optional(), + }) + .strict(); + +export const REVIEW_OUTPUT_POLICY_SCHEMA = z + .object({ + includeUsage: z.boolean(), + recordPayloadTelemetry: z.boolean(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_LEVEL_SCHEMA = z.enum(['info', 'warn', 'error']); + +export const REVIEW_SCORE_COMPONENT_SCHEMA = z + .object({ + id: z.string().min(1), + scoreText: z.string(), + score: z.number(), + weight: z.number().optional(), + }) + .strict(); + +export const REVIEW_SCORE_SCHEMA = z + .object({ + ruleId: z.string().min(1), + score: z.number(), + scoreText: z.string(), + severity: REVIEW_SEVERITY_SCHEMA, + findingCount: z.number().int().nonnegative().optional(), + components: z.array(REVIEW_SCORE_COMPONENT_SCHEMA).optional(), + }) + .strict(); + +export const REVIEW_FINDING_SCHEMA = z + .object({ + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + severity: REVIEW_SEVERITY_SCHEMA, + message: z.string(), + line: z.number().int().positive(), + column: z.number().int().positive(), + match: z.string(), + analysis: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + }) + .strict(); + +export const REVIEW_DIAGNOSTIC_SCHEMA = z + .object({ + level: REVIEW_DIAGNOSTIC_LEVEL_SCHEMA, + code: z.string().min(1), + message: z.string(), + ruleId: z.string().optional(), + context: z.record(z.string(), z.unknown()).optional(), + }) + .strict(); + +export const REVIEW_USAGE_SCHEMA = z + .object({ + inputTokens: z.number().int().nonnegative().optional(), + outputTokens: z.number().int().nonnegative().optional(), + modelCalls: z.number().int().nonnegative(), + costUsd: z.number().nonnegative().optional(), + wallClockMs: z.number().int().nonnegative().optional(), + }) + .strict(); + +export const REVIEW_RESULT_SCHEMA = z + .object({ + findings: z.array(REVIEW_FINDING_SCHEMA), + scores: z.array(REVIEW_SCORE_SCHEMA), + diagnostics: z.array(REVIEW_DIAGNOSTIC_SCHEMA), + usage: REVIEW_USAGE_SCHEMA.optional(), + hadOperationalErrors: z.boolean().optional(), + }) + .strict(); + +export const REVIEW_REQUEST_SCHEMA = z + .object({ + target: REVIEW_TARGET_SCHEMA, + rules: z.array(REVIEW_RULE_SCHEMA).min(1), + context: z.array(REVIEW_CONTEXT_SCHEMA).optional(), + budget: REVIEW_BUDGET_SCHEMA, + outputPolicy: REVIEW_OUTPUT_POLICY_SCHEMA, + modelCall: z.enum(['single', 'agent', 'auto']), + }) + .strict(); diff --git a/src/review/types.ts b/src/review/types.ts new file mode 100644 index 00000000..97e22039 --- /dev/null +++ b/src/review/types.ts @@ -0,0 +1,192 @@ +/** + * Neutral review-domain contract types for the VectorLint harness. + * + * Everything an executor needs to review target content against + * source-backed rules flows through this module. Callers build a + * {@link ReviewRequest} and executors return a {@link ReviewResult}. + * + * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + * + * This module is implementation-neutral: it deliberately exposes no legacy + * scoring-mode, rubric, or model-authored rule-override surface. `modelCall` + * selects how the reviewer model is invoked, not how rules are scored. + */ + +/** Finding/Rule severity. */ +export type ReviewSeverity = 'error' | 'warning'; + +/** Objective Via Negativa condition that counts as a violation when present. */ +export interface ReviewViolationCondition { + id: string; + description: string; +} + +/** + * Target content under review. The on-page boundary (audit Finding #5) is + * enforced against this: executors may only read sections of `content`. + */ +export interface ReviewTarget { + /** Stable absolute URI (file:// or virtual scheme for in-memory content). */ + uri: string; + /** Full target content. May be paged by an agent executor. */ + content: string; + /** MIME-ish content type, e.g. 'text/markdown'. */ + contentType: string; + /** Optional byte length hint; computed from content if absent. */ + byteLength?: number; +} + +/** + * A source-backed, caller-authored rule. Model-authored rule overrides are + * explicitly disallowed (audit Finding #1, #5). + */ +export interface ReviewRule { + /** Stable id, formatted Pack.Rule[.Criterion] in output. */ + id: string; + /** Canonical source path (where the rule body came from). */ + source: string; + /** The rule prompt body. Source-backed; never model-authored. */ + body: string; + /** Human-readable name. */ + name?: string; + /** 'error' | 'warning'. Defaults to 'warning'. */ + severity?: ReviewSeverity; + /** Optional structured Via Negativa violation conditions. */ + violationConditions?: ReviewViolationCondition[]; +} + +/** + * Caller-supplied context. Explicitly in scope (audit boundary rule): + * VectorLint does NOT discover workspace files as context on its own; the + * caller owns exploration and context gathering. + */ +export interface ReviewContext { + /** Short label used in diagnostics/tracing. */ + label: string; + /** The context content itself. */ + content: string; + /** How this context relates to the target, e.g. 'reference' | 'glossary'. */ + relation?: string; + /** Optional source URI for provenance. */ + uri?: string; +} + +/** + * The on-page boundary scope: the normalized URIs an executor may read. + * Built by {@link buildScope} (see boundary.ts) and checked by + * {@link isInScope}. + */ +export interface ReviewScope { + /** Normalized absolute URIs that are in scope (target + caller context). */ + readonly allowedUris: ReadonlySet; +} + +/** + * Hard bounds on a single review (audit Finding #7). These limit work, not + * output. Executors MUST check these and surface a ReviewDiagnostic (or fail + * the run) when exceeded. + */ +export interface ReviewBudget { + maxTargetBytes: number; + maxCallerContextBytes: number; + maxChunksPerRule: number; + maxModelCallsPerReview: number; + /** Maximum elapsed review time in milliseconds. This is the run timeout. */ + maxWallClockMs: number; +} + +/** + * Controls optional output/telemetry behavior. Diagnostics are always part of + * ReviewResult; this policy does not hide operational warnings. Audit + * Finding #9: payload telemetry must be a separate opt-in from metadata + * telemetry. + */ +export interface ReviewOutputPolicy { + /** Include usage/cost in the result. */ + includeUsage: boolean; + /** Opt-in: record prompt/content payloads to telemetry. Default false. */ + recordPayloadTelemetry: boolean; +} + +/** A verified finding anchored in the target content. */ +export interface ReviewFinding { + ruleId: string; + ruleSource: string; + severity: ReviewSeverity; + message: string; + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. Unverified finding evidence is a diagnostic. */ + match: string; + analysis?: string; + suggestion?: string; + fix?: string; +} + +/** A per-rule score produced through shared finding processing. */ +export interface ReviewScore { + ruleId: string; + score: number; + /** Human-readable score, e.g. "8.0/10". */ + scoreText: string; + severity: ReviewSeverity; + /** Count of verified findings that contributed to this score, if applicable. */ + findingCount?: number; + components?: ReviewScoreComponent[]; +} + +export interface ReviewScoreComponent { + id: string; + scoreText: string; + score: number; + weight?: number; +} + +export type ReviewDiagnosticLevel = 'info' | 'warn' | 'error'; + +/** An operational or finding-processing note. Always part of ReviewResult. */ +export interface ReviewDiagnostic { + level: ReviewDiagnosticLevel; + /** Stable machine code, e.g. 'finding-evidence-not-locatable'. */ + code: string; + message: string; + ruleId?: string; + /** Extra machine-readable detail (model-call count, evidence preview, ...). */ + context?: Record; +} + +/** Aggregated resource usage for a review run. */ +export interface ReviewUsage { + inputTokens?: number; + outputTokens?: number; + modelCalls: number; + costUsd?: number; + wallClockMs?: number; +} + +/** The output from any executor, produced through shared finding processing. */ +export interface ReviewResult { + findings: ReviewFinding[]; + scores: ReviewScore[]; + diagnostics: ReviewDiagnostic[]; + usage?: ReviewUsage; + /** True if the run hit an operational error but still returned partial results. */ + hadOperationalErrors?: boolean; +} + +/** How to call the reviewer model. 'auto' resolves via chooseModelCall(). */ +export type ReviewModelCall = 'single' | 'agent' | 'auto'; + +/** The complete input to any executor. */ +export interface ReviewRequest { + target: ReviewTarget; + rules: ReviewRule[]; + /** Caller-supplied, in-scope context (never workspace-discovered). */ + context?: ReviewContext[]; + budget: ReviewBudget; + outputPolicy: ReviewOutputPolicy; + /** Reviewer model call shape. 'auto' resolves via chooseModelCall(). */ + modelCall: ReviewModelCall; +} diff --git a/tests/review/boundary.test.ts b/tests/review/boundary.test.ts new file mode 100644 index 00000000..5ff92dad --- /dev/null +++ b/tests/review/boundary.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { buildScope, isInScope, normalizeReviewUri } from '../../src/review'; + +describe('review boundary', () => { + const scope = buildScope({ + targetUri: 'file:///repo/docs/guide.md', + contextUris: ['file:///repo/docs/glossary.md'], + }); + + it('target is always in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/guide.md')).toBe(true); + }); + + it('caller-supplied context uri is in scope', () => { + expect(isInScope(scope, 'file:///repo/docs/glossary.md')).toBe(true); + }); + + it('arbitrary workspace file is NOT in scope', () => { + expect(isInScope(scope, 'file:///repo/src/index.ts')).toBe(false); + }); + + it('rejects path traversal that escapes the target', () => { + expect(isInScope(scope, 'file:///repo/docs/../src/index.ts')).toBe(false); + }); + + it('normalizes traversal that resolves back onto an in-scope uri', () => { + expect(isInScope(scope, 'file:///repo/docs/sub/../guide.md')).toBe(true); + }); +}); + +describe('normalizeReviewUri', () => { + it('resolves dot segments in file uris', () => { + expect(normalizeReviewUri('file:///repo/a/../b/./c.md')).toBe('file:///repo/b/c.md'); + }); + + it('preserves authority', () => { + expect(normalizeReviewUri('file://host/x/../y.md')).toBe('file://host/y.md'); + }); +}); diff --git a/tests/review/budget.test.ts b/tests/review/budget.test.ts new file mode 100644 index 00000000..a63bcc24 --- /dev/null +++ b/tests/review/budget.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { + BudgetExceededError, + DEFAULT_REVIEW_BUDGET, + REVIEW_BUDGET_SCHEMA, + enforceBudget, +} from '../../src/review'; +import { VectorlintError } from '../../src/errors'; + +describe('review budget defaults', () => { + it('provides sensible positive defaults', () => { + expect(DEFAULT_REVIEW_BUDGET.maxModelCallsPerReview).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxWallClockMs).toBeGreaterThan(0); + expect(DEFAULT_REVIEW_BUDGET.maxTargetBytes).toBeGreaterThan(0); + }); + + it('does not cap findings or headless retries', () => { + expect('maxFindingsPerRule' in DEFAULT_REVIEW_BUDGET).toBe(false); + expect('maxHeadlessRetries' in DEFAULT_REVIEW_BUDGET).toBe(false); + }); +}); + +describe('REVIEW_BUDGET_SCHEMA', () => { + it('applies defaults for omitted fields', () => { + const parsed = REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 5 }); + expect(parsed.maxWallClockMs).toBe(DEFAULT_REVIEW_BUDGET.maxWallClockMs); + expect(parsed.maxModelCallsPerReview).toBe(5); + }); + + it('rejects non-positive limits', () => { + expect(() => REVIEW_BUDGET_SCHEMA.parse({ maxModelCallsPerReview: 0 })).toThrow(); + }); +}); + +describe('enforceBudget', () => { + it('is silent within limits', () => { + expect(() => + enforceBudget(DEFAULT_REVIEW_BUDGET, { modelCalls: 1, elapsedMs: 1000 }), + ).not.toThrow(); + }); + + it('throws BudgetExceededError when model calls exceed the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxModelCallsPerReview: 3 }, + { modelCalls: 4, elapsedMs: 0 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('throws BudgetExceededError when wall clock exceeds the limit', () => { + expect(() => + enforceBudget( + { ...DEFAULT_REVIEW_BUDGET, maxWallClockMs: 100 }, + { modelCalls: 0, elapsedMs: 101 }, + ), + ).toThrow(BudgetExceededError); + }); + + it('extends the repository error base', () => { + const err = new BudgetExceededError('boom', 'maxModelCallsPerReview', 9); + expect(err).toBeInstanceOf(VectorlintError); + expect(err.limit).toBe('maxModelCallsPerReview'); + expect(err.actual).toBe(9); + expect(err.code).toBe('BUDGET_EXCEEDED'); + }); +}); diff --git a/tests/review/executor.test.ts b/tests/review/executor.test.ts new file mode 100644 index 00000000..c15e670b --- /dev/null +++ b/tests/review/executor.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_MODEL_CALLS, + chooseModelCall, + type ReviewExecutor, + type ReviewRequest, +} from '../../src/review'; + +describe('REVIEW_MODEL_CALLS', () => { + it('exposes single | agent | auto', () => { + expect(REVIEW_MODEL_CALLS).toEqual(['single', 'agent', 'auto']); + }); +}); + +describe('chooseModelCall', () => { + it('respects an explicit single call', () => { + expect(chooseModelCall('single', { targetBytes: 2_000_000, rules: 10 })).toBe('single'); + }); + + it('respects an explicit agent call', () => { + expect(chooseModelCall('agent', { targetBytes: 10, rules: 1 })).toBe('agent'); + }); + + it('returns single for small inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 10_000, rules: 1 })).toBe('single'); + }); + + it('returns agent for large inputs under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 2_000_000, rules: 1 })).toBe('agent'); + }); + + it('returns agent for many rules under auto', () => { + expect(chooseModelCall('auto', { targetBytes: 1_000, rules: 6 })).toBe('agent'); + }); +}); + +describe('ReviewExecutor contract', () => { + it('can be implemented and run() returns a ReviewResult', async () => { + const fake: ReviewExecutor = { + run: () => Promise.resolve({ findings: [], scores: [], diagnostics: [] }), + }; + const result = await fake.run({} as unknown as ReviewRequest); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + }); +}); diff --git a/tests/review/module-surface.test.ts b/tests/review/module-surface.test.ts new file mode 100644 index 00000000..6c03ad30 --- /dev/null +++ b/tests/review/module-surface.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import * as review from '../../src/review'; + +describe('src/review public surface', () => { + it('exports the core contract values and functions', () => { + // Runtime presence checks for values; types are checked by tsc. + expect(review.REVIEW_MODEL_CALLS).toBeDefined(); + expect(review.DEFAULT_REVIEW_BUDGET).toBeDefined(); + expect(typeof review.chooseModelCall).toBe('function'); + expect(typeof review.isInScope).toBe('function'); + expect(typeof review.buildReviewRequest).toBe('function'); + }); + + it('exposes the boundary, budget, and error helpers', () => { + expect(typeof review.buildScope).toBe('function'); + expect(typeof review.enforceBudget).toBe('function'); + expect(typeof review.normalizeReviewUri).toBe('function'); + expect(review.BudgetExceededError).toBeDefined(); + }); + + it('exposes the Zod schemas for every external shape', () => { + expect(review.REVIEW_TARGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_RULE_SCHEMA).toBeDefined(); + expect(review.REVIEW_CONTEXT_SCHEMA).toBeDefined(); + expect(review.REVIEW_BUDGET_SCHEMA).toBeDefined(); + expect(review.REVIEW_OUTPUT_POLICY_SCHEMA).toBeDefined(); + expect(review.REVIEW_FINDING_SCHEMA).toBeDefined(); + expect(review.REVIEW_SCORE_SCHEMA).toBeDefined(); + expect(review.REVIEW_DIAGNOSTIC_SCHEMA).toBeDefined(); + expect(review.REVIEW_USAGE_SCHEMA).toBeDefined(); + expect(review.REVIEW_RESULT_SCHEMA).toBeDefined(); + expect(review.REVIEW_REQUEST_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts new file mode 100644 index 00000000..618efa03 --- /dev/null +++ b/tests/review/request-builder.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest'; +import { + DEFAULT_REVIEW_BUDGET, + REVIEW_RULE_SCHEMA, + buildReviewRequest, +} from '../../src/review'; +import { ValidationError } from '../../src/errors'; +import { Severity } from '../../src/evaluators/types'; +import type { PromptFile } from '../../src/schemas/prompt-schemas'; +import type { ReviewContext, ReviewTarget } from '../../src/review'; + +function makePrompt(overrides: Partial = {}): PromptFile { + return { + id: 'pseudo-advice', + filename: 'pseudo-advice.md', + fullPath: '/repo/presets/VectorLint/pseudo-advice.md', + meta: { + id: 'PseudoAdvice', + name: 'Pseudo Advice', + severity: Severity.WARNING, + type: 'check', + criteria: [{ id: 'Vague', name: 'Vague advice' }], + }, + body: 'Check for pseudo advice.', + pack: 'VectorLint', + ...overrides, + }; +} + +describe('buildReviewRequest', () => { + const target: ReviewTarget = { + uri: 'file:///repo/docs/guide.md', + content: '# Guide', + contentType: 'text/markdown', + }; + + it('maps one PromptFile to one ReviewRule with default budget and auto modelCall', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + expect(request.modelCall).toBe('auto'); + expect(request.budget).toEqual(DEFAULT_REVIEW_BUDGET); + expect(request.rules).toHaveLength(1); + + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule?.id).toBe('VectorLint.PseudoAdvice'); + expect(rule?.source).toBe('/repo/presets/VectorLint/pseudo-advice.md'); + expect(rule?.body).toBe('Check for pseudo advice.'); + expect(rule?.severity).toBe('warning'); + expect(rule?.name).toBe('Pseudo Advice'); + // Strict parse proves no legacy evaluator/criteria/type fields leaked. + expect(() => REVIEW_RULE_SCHEMA.parse(rule)).not.toThrow(); + }); + + it('does not copy legacy meta.type, evaluator, or criteria fields', () => { + const request = buildReviewRequest({ target, prompts: [makePrompt()] }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'type' in rule).toBe(false); + expect(rule && 'evaluator' in rule).toBe(false); + expect(rule && 'criteria' in rule).toBe(false); + }); + + it('throws a ValidationError when no prompts are supplied', () => { + expect(() => buildReviewRequest({ target, prompts: [] })).toThrow(ValidationError); + }); + + it('passes caller-supplied context through unchanged', () => { + const context: ReviewContext[] = [ + { label: 'glossary', content: 'term a means ...', relation: 'reference' }, + ]; + const request = buildReviewRequest({ target, prompts: [makePrompt()], context }); + expect(request.context).toBe(context); + }); + + it('honors an explicit modelCall override', () => { + const request = buildReviewRequest({ + target, + prompts: [makePrompt()], + config: { modelCall: 'agent' }, + }); + expect(request.modelCall).toBe('agent'); + }); + + it('omits severity on the rule when the prompt has none', () => { + const request = buildReviewRequest({ + target, + prompts: [ + makePrompt({ + meta: { id: 'PseudoAdvice', name: 'Pseudo Advice' }, + }), + ], + }); + const rule = request.rules[0]; + expect(rule).toBeDefined(); + expect(rule && 'severity' in rule).toBe(false); + }); +}); diff --git a/tests/review/result.test.ts b/tests/review/result.test.ts new file mode 100644 index 00000000..3be3767a --- /dev/null +++ b/tests/review/result.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest'; +import { + REVIEW_DIAGNOSTIC_SCHEMA, + REVIEW_FINDING_SCHEMA, + REVIEW_RESULT_SCHEMA, +} from '../../src/review'; + +describe('ReviewFinding schema', () => { + it('parses a complete finding', () => { + const finding = REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'VectorLint.Consistency', + ruleSource: '/repo/presets/VectorLint/consistency.md', + severity: 'warning', + message: 'Vague advice.', + line: 5, + column: 1, + match: 'consider leveraging', + analysis: 'too vague', + suggestion: 'be specific', + fix: 'consider using concrete terms', + }); + expect(finding.line).toBe(5); + expect(finding.match).toBe('consider leveraging'); + expect(finding.severity).toBe('warning'); + }); + + it('rejects a finding with an unknown severity', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'critical', + message: 'x', + line: 1, + column: 1, + match: 'x', + }), + ).toThrow(); + }); + + it('rejects a finding missing required anchored match evidence', () => { + expect(() => + REVIEW_FINDING_SCHEMA.parse({ + ruleId: 'P.R', + ruleSource: '/x.md', + severity: 'warning', + message: 'x', + line: 1, + column: 1, + }), + ).toThrow(); + }); +}); + +describe('ReviewDiagnostic schema', () => { + it('parses a diagnostic with level/code/message', () => { + const diag = REVIEW_DIAGNOSTIC_SCHEMA.parse({ + level: 'warn', + code: 'finding-evidence-not-locatable', + message: 'could not anchor the finding', + }); + expect(diag.level).toBe('warn'); + }); + + it('rejects an unknown level', () => { + expect(() => + REVIEW_DIAGNOSTIC_SCHEMA.parse({ level: 'fatal', code: 'x', message: 'y' }), + ).toThrow(); + }); +}); + +describe('ReviewResult schema', () => { + it('accepts findings/scores/diagnostics and defaults usage to undefined', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [], diagnostics: [] }); + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + expect(result.usage).toBeUndefined(); + }); + + it('keeps diagnostics mandatory', () => { + expect(() => + REVIEW_RESULT_SCHEMA.parse({ findings: [], scores: [] }), + ).toThrow(); + }); + + it('accepts usage when provided', () => { + const result = REVIEW_RESULT_SCHEMA.parse({ + findings: [], + scores: [], + diagnostics: [], + usage: { modelCalls: 2, inputTokens: 10, outputTokens: 5 }, + }); + expect(result.usage?.modelCalls).toBe(2); + }); +}); diff --git a/tests/review/types.test.ts b/tests/review/types.test.ts new file mode 100644 index 00000000..d3bee632 --- /dev/null +++ b/tests/review/types.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { REVIEW_CONTEXT_SCHEMA, REVIEW_RULE_SCHEMA, REVIEW_TARGET_SCHEMA } from '../../src/review'; + +describe('ReviewTarget schema', () => { + it('parses a valid target', () => { + const target = REVIEW_TARGET_SCHEMA.parse({ + uri: 'file:///repo/docs/guide.md', + content: '# Guide\n\nBody text.', + contentType: 'text/markdown', + }); + expect(target.uri).toBe('file:///repo/docs/guide.md'); + }); + + it('allows empty content (streaming target)', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ uri: 'file:///x', content: '', contentType: 'text/plain' }), + ).not.toThrow(); + }); + + it('rejects a target missing a uri', () => { + expect(() => + REVIEW_TARGET_SCHEMA.parse({ content: 'x', contentType: 'text/plain' }), + ).toThrow(); + }); +}); + +describe('ReviewRule schema', () => { + it('parses a rule with required fields and defaults severity to warning', () => { + const rule = REVIEW_RULE_SCHEMA.parse({ + id: 'Consistency', + source: 'VectorLint/consistency.md', + body: 'Check internal consistency.', + violationConditions: [ + { id: 'Contradiction', description: 'The target makes mutually inconsistent claims.' }, + ], + }); + expect(rule.severity).toBe('warning'); + expect(rule.violationConditions?.[0]?.id).toBe('Contradiction'); + }); + + it('rejects legacy evaluator and judge criteria fields', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'Judge whether the tone is good.', + evaluator: 'judge', + criteria: [{ id: 'Tone', name: 'Tone quality' }], + }), + ).toThrow(); + }); + + it('rejects an unknown severity', () => { + expect(() => + REVIEW_RULE_SCHEMA.parse({ + id: 'Tone', + source: 'VectorLint/tone.md', + body: 'x', + severity: 'critical', + }), + ).toThrow(); + }); +}); + +describe('ReviewContext schema', () => { + it('parses caller-supplied scoped content', () => { + const ctx = REVIEW_CONTEXT_SCHEMA.parse({ + label: 'related-glossary', + content: 'Term A means ...', + relation: 'reference', + }); + expect(ctx.label).toBe('related-glossary'); + }); +}); From e4259e7865a1604a4091d6432bcc04c516b9d2ee Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:11:06 +0100 Subject: [PATCH 10/24] Extract single finding evidence verifier - Add src/findings/finding-evidence-verifier.ts wrapping locateQuotedText from src/output/location.ts as the single source of truth (audit #6). - Anchored quotes return verified line/column/match; unanchored quotes return a finding-evidence-not-locatable warn diagnostic and no finding. - Never falls back to a model-provided line number the way the old agent path did. - Cover anchored, unanchored, no-line-hint, and truncation cases. --- src/findings/finding-evidence-verifier.ts | 84 +++++++++++++++++++ .../finding-evidence-verifier.test.ts | 69 +++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/findings/finding-evidence-verifier.ts create mode 100644 tests/findings/finding-evidence-verifier.test.ts diff --git a/src/findings/finding-evidence-verifier.ts b/src/findings/finding-evidence-verifier.ts new file mode 100644 index 00000000..7a6ffb6a --- /dev/null +++ b/src/findings/finding-evidence-verifier.ts @@ -0,0 +1,84 @@ +import { locateQuotedText, type LocationWithMatch } from '../output/location'; + +/** Stable machine code emitted when finding evidence cannot be anchored. */ +export const FINDING_EVIDENCE_NOT_LOCATABLE = 'finding-evidence-not-locatable' as const; + +/** Evidence markers a model returns for a candidate finding. */ +export interface FindingEvidenceInput { + quoted_text: string; + context_before?: string; + context_after?: string; + /** Optional 1-based line hint from the model; never used as a fallback. */ + line?: number; +} + +/** Located coordinates for a verified finding, anchored in target content. */ +export interface VerifiedFindingCoords { + /** 1-based line in the target content. */ + line: number; + /** 1-based column. */ + column: number; + /** Verified anchored text. */ + match: string; +} + +/** Operational note produced when evidence cannot be verified. */ +export interface FindingEvidenceDiagnostic { + code: string; + level: 'warn' | 'error'; + message: string; +} + +/** The outcome of verifying a single candidate finding's evidence. */ +export interface FindingEvidenceVerification { + verified: boolean; + finding?: VerifiedFindingCoords; + diagnostic?: FindingEvidenceDiagnostic; +} + +/** + * The single source of truth for finding evidence verification + * (audit Finding #6). + * + * Wraps the existing `locateQuotedText` fuzzy matcher; it never reimplements + * matching. On a successful anchor it returns a verified finding with the + * located line/column/match. When the quoted text cannot be anchored it + * returns a `finding-evidence-not-locatable` warn diagnostic and NO finding — + * it never falls back to a model-provided line number the way the old agent + * path did. + */ +export function verifyFindingEvidence( + content: string, + findingEvidence: FindingEvidenceInput, +): FindingEvidenceVerification { + const location: LocationWithMatch | null = locateQuotedText( + content, + { + quoted_text: findingEvidence.quoted_text || '', + context_before: findingEvidence.context_before || '', + context_after: findingEvidence.context_after || '', + }, + 80, + findingEvidence.line, + ); + + if (!location) { + return { + verified: false, + diagnostic: { + code: FINDING_EVIDENCE_NOT_LOCATABLE, + level: 'warn', + message: `Could not locate quoted text "${(findingEvidence.quoted_text || '').slice(0, 60)}" in target content.`, + }, + }; + } + + return { + verified: true, + finding: { + line: location.line, + column: location.column, + match: location.match, + }, + }; +} diff --git a/tests/findings/finding-evidence-verifier.test.ts b/tests/findings/finding-evidence-verifier.test.ts new file mode 100644 index 00000000..64d38209 --- /dev/null +++ b/tests/findings/finding-evidence-verifier.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, +} from '../../src/findings/finding-evidence-verifier'; + +const CONTENT = 'line one\nline two has the quote\nline three'; + +describe('verifyFindingEvidence', () => { + it('locates an exact quote and returns a verified finding', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'the quote', + context_before: '', + context_after: '', + line: 2, + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(2); + expect(result.finding?.match).toBe('the quote'); + expect(result.diagnostic).toBeUndefined(); + }); + + it('returns a warn diagnostic (not a fallback finding) when evidence cannot be located', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'this does not exist anywhere', + context_before: '', + context_after: '', + line: 99, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + expect(result.diagnostic?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostic?.level).toBe('warn'); + }); + + it('never returns a finding whose line came from the model when the quote did not anchor', () => { + // The old agent path fell back to the model-provided line (here 1) and + // surfaced the raw quoted_text as a verified match. Assert we do not. + // (Use a lexically distant quote that locateQuotedText cannot anchor.) + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'definitely not present xyzzy', + context_before: '', + context_after: '', + line: 1, + }); + expect(result.verified).toBe(false); + expect(result.finding).toBeUndefined(); + }); + + it('still anchors a quote when no line hint is provided', () => { + const result = verifyFindingEvidence(CONTENT, { + quoted_text: 'line three', + }); + expect(result.verified).toBe(true); + expect(result.finding?.line).toBe(3); + expect(result.finding?.column).toBe(1); + }); + + it('produces a diagnostic that mentions the (truncated) quoted text', () => { + const longQuote = 'x'.repeat(120); + const result = verifyFindingEvidence(CONTENT, { + quoted_text: longQuote, + }); + expect(result.verified).toBe(false); + expect(result.diagnostic?.message).toContain('x'.repeat(60)); + // Truncated, not the full 120 chars. + expect(result.diagnostic?.message).not.toContain('x'.repeat(61)); + }); +}); From 25ec49aa43ec112c56785ef41a20207ebb4c9c88 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:13:50 +0100 Subject: [PATCH 11/24] Add shared finding severity and scoring helpers - Add src/findings/types.ts with the FindingProcessingInput contract and strict Zod schemas; no modelCall/mode/agent/evaluator/judge/rubric fields, so legacy judge/evaluator-shaped input is rejected. - Add src/findings/severity.ts: resolveSeverity derives severity from the calculateCheckScore result (one path, no agent stamping); buildRuleId matches Pack.Rule[.Criterion] naming without importing src/agent/. - Add src/findings/scorer.ts as a thin wrapper over calculateCheckScore; the verified finding count drives the score. - Cover schema parse/reject, severity, rule-id, criterion resolution, and score numerics against calculateCheckScore. --- src/findings/scorer.ts | 60 ++++++++++++ src/findings/severity.ts | 52 +++++++++++ src/findings/types.ts | 160 ++++++++++++++++++++++++++++++++ tests/findings/scorer.test.ts | 94 +++++++++++++++++++ tests/findings/severity.test.ts | 73 +++++++++++++++ tests/findings/types.test.ts | 145 +++++++++++++++++++++++++++++ 6 files changed, 584 insertions(+) create mode 100644 src/findings/scorer.ts create mode 100644 src/findings/severity.ts create mode 100644 src/findings/types.ts create mode 100644 tests/findings/scorer.test.ts create mode 100644 tests/findings/severity.test.ts create mode 100644 tests/findings/types.test.ts diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts new file mode 100644 index 00000000..b5fb93fa --- /dev/null +++ b/src/findings/scorer.ts @@ -0,0 +1,60 @@ +import { calculateCheckScore, type CheckScoringOptions } from '../scoring'; +import type { CheckResult } from '../prompts/schema'; +import { resolveSeverity } from './severity'; +import type { RawViolation, RuleSeverity } from './types'; + +/** Optional scoring knobs mirroring {@link CheckScoringOptions}. */ +export interface ScoreCheckOptions { + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +} + +/** + * The count/density score result for a rule's verified findings. Mirrors the + * fields the processor needs to assemble a `ReviewScore`. The numeric values + * come straight from {@link calculateCheckScore}; this layer never + * reimplements the scoring math (audit Finding #4). + */ +export interface ScoredCheck { + score: number; + scoreText: string; + severity: RuleSeverity; + /** Count of verified findings that drove the score. */ + findingCount: number; + /** The raw check result, used to resolve severity. */ + scored: CheckResult; +} + +/** + * Scores a rule's verified findings using the existing check density formula. + * + * The score is driven by the **verified** finding count, not the raw candidate + * count (audit Finding #6): callers must pass only findings whose evidence has + * been anchored. This is a thin adapter over {@link calculateCheckScore}; it + * only normalizes the result into the shared score shape. + */ +export function scoreCheck(params: { + verifiedViolations: RawViolation[]; + wordCount: number; + strictness?: CheckScoringOptions['strictness']; + promptSeverity?: RuleSeverity; +}): ScoredCheck { + const scored = calculateCheckScore( + params.verifiedViolations, + params.wordCount, + { + ...(params.strictness !== undefined ? { strictness: params.strictness } : {}), + ...(params.promptSeverity !== undefined + ? { promptSeverity: params.promptSeverity } + : {}), + }, + ); + + return { + score: scored.final_score, + scoreText: `${scored.final_score.toFixed(1)}/10`, + severity: resolveSeverity({ scored }), + findingCount: scored.violation_count, + scored, + }; +} diff --git a/src/findings/severity.ts b/src/findings/severity.ts new file mode 100644 index 00000000..cca534f0 --- /dev/null +++ b/src/findings/severity.ts @@ -0,0 +1,52 @@ +import type { CheckResult } from '../prompts/schema'; +import type { FindingsCriterion, RuleSeverity } from './types'; + +/** Input to {@link resolveSeverity}. */ +export interface SeverityInput { + scored: CheckResult; +} + +/** + * The one severity resolution path for objective violation checks + * (audit Finding #4). Severity is derived from the existing count/density + * score result (`CheckResult.severity`); there is no agent-specific or + * mode-specific severity stamping here. + */ +export function resolveSeverity(input: SeverityInput): RuleSeverity { + return input.scored.severity; +} + +/** + * Builds the hierarchical output rule id matching the current standard + * orchestrator naming (`buildRuleName`): `Pack.Rule` or, when a violation is + * attributed to a criterion, `Pack.Rule.Criterion`. This is a faithful port of + * that behavior; it does not import from `src/agent/`. + */ +export function buildRuleId( + pack: string, + ruleId: string, + criterionId?: string, +): string { + const parts = [pack, ruleId]; + if (criterionId) { + parts.push(criterionId); + } + return parts.join('.'); +} + +/** + * Resolves a criterion id from the rule's declared criteria by matching a + * `criterionName` a violation carries to a declared criterion's `name`. + * Returns `undefined` when the violation is not attributed to a declared + * criterion (the common case for objective check violations). + */ +export function resolveCriterionId( + criteria: readonly FindingsCriterion[] | undefined, + criterionName: string | undefined, +): string | undefined { + if (!criterionName || !criteria) { + return undefined; + } + const match = criteria.find((c) => c.name === criterionName); + return match?.id; +} diff --git a/src/findings/types.ts b/src/findings/types.ts new file mode 100644 index 00000000..9a6360c6 --- /dev/null +++ b/src/findings/types.ts @@ -0,0 +1,160 @@ +import { z } from 'zod'; +import { Severity } from '../evaluators/types'; + +/** + * Input contract for shared finding processing (Phase 3, audit Finding #6). + * + * This module is the boundary between "what a model returned" (raw candidate + * violations) and "what the formatters receive" (`ReviewResult`). It is + * intentionally independent of model call (`single` | `agent`): there is no + * `modelCall`, `mode`, `agent`, `evaluator`, `judge`, or rubric field here. + * + * Following the `src/review/` pattern, the domain types below are explicit + * interfaces (so their optional properties stay assignable under + * `exactOptionalPropertyTypes` to `CheckItem`/`GateViolationLike`), and the + * Zod schemas mirror them for boundary validation. The schemas are strict so + * legacy `evaluator`/`judge`/`modelCall`/`mode`/`agent`/rubric payloads are + * rejected. + */ + +/** + * The two severity levels objective violation checks can resolve to. Mirrors + * `ReviewSeverity`; values are identical (`'error'` | `'warning'`). + */ +export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; + +/** Optional strictness knob for check density scoring. */ +export type Strictness = number | 'lenient' | 'standard' | 'strict'; + +/** + * The six boolean evidence gates a model returns per violation. Structurally + * identical to `GateChecks` in `src/prompts/schema.ts`. + */ +export interface GateChecks { + rule_supports_claim: boolean; + evidence_exact: boolean; + context_supports_violation: boolean; + plausible_non_violation: boolean; + fix_is_drop_in: boolean; + fix_preserves_meaning: boolean; +} + +/** + * A raw candidate finding returned by a reviewer model, before evidence + * verification or filtering. `analysis` is the only required descriptive + * field; everything else is optional to tolerate partial model output while + * still failing closed at the filter and verifier stages. + */ +export interface RawViolation { + line?: number; + quoted_text?: string; + context_before?: string; + context_after?: string; + description?: string; + /** Optional criterion attribution used to build `Pack.Rule.Criterion` ids. */ + criterionName?: string; + analysis: string; + message?: string; + suggestion?: string; + fix?: string; + rule_quote?: string; + confidence?: number; + checks?: GateChecks; +} + +/** + * Criterion metadata used ONLY to name output rule ids + * (`Pack.Rule.Criterion`). Weight and target are judge/rubric concerns and are + * deliberately absent so the finding-processing contract cannot leak them. + */ +export interface FindingsCriterion { + id: string; + name: string; +} + +/** + * The narrow slice of prompt metadata that finding processing needs: severity + * for scoring, strictness for density, and criteria for rule-id naming. It + * excludes `evaluator`, `type`, `evaluateAs`, `target`, and other legacy + * fields that do not affect objective violation-check processing. + */ +export interface PromptMetaForFindings { + severity?: RuleSeverity; + strictness?: Strictness; + criteria?: FindingsCriterion[]; +} + +/** + * The full finding-processing input. There is deliberately no `outputFormat`: + * the processor emits a format-agnostic `ReviewResult`; the caller routes it + * to a formatter. + */ +export interface FindingProcessingInput { + pack: string; + ruleId: string; + ruleSource: string; + candidateFindings: RawViolation[]; + wordCount: number; + promptMeta: PromptMetaForFindings; + targetContent: string; +} + +// --- Boundary validation schemas (mirror the interfaces above) --------------- + +export const GATE_CHECKS_SCHEMA = z + .object({ + rule_supports_claim: z.boolean(), + evidence_exact: z.boolean(), + context_supports_violation: z.boolean(), + plausible_non_violation: z.boolean(), + fix_is_drop_in: z.boolean(), + fix_preserves_meaning: z.boolean(), + }) + .strict(); + +export const RAW_VIOLATION_SCHEMA = z + .object({ + line: z.number().optional(), + quoted_text: z.string().optional(), + context_before: z.string().optional(), + context_after: z.string().optional(), + description: z.string().optional(), + criterionName: z.string().optional(), + analysis: z.string(), + message: z.string().optional(), + suggestion: z.string().optional(), + fix: z.string().optional(), + rule_quote: z.string().optional(), + confidence: z.number().optional(), + checks: GATE_CHECKS_SCHEMA.optional(), + }) + .strict(); + +export const FINDINGS_CRITERION_SCHEMA = z + .object({ + id: z.string().min(1), + name: z.string().min(1), + }) + .strict(); + +export const PROMPT_META_FOR_FINDINGS_SCHEMA = z + .object({ + severity: z.nativeEnum(Severity).optional(), + strictness: z + .union([z.number().positive(), z.enum(['lenient', 'standard', 'strict'])]) + .optional(), + criteria: z.array(FINDINGS_CRITERION_SCHEMA).optional(), + }) + .strict(); + +export const FINDING_PROCESSING_INPUT_SCHEMA = z + .object({ + pack: z.string().min(1), + ruleId: z.string().min(1), + ruleSource: z.string().min(1), + candidateFindings: z.array(RAW_VIOLATION_SCHEMA), + wordCount: z.number().nonnegative(), + promptMeta: PROMPT_META_FOR_FINDINGS_SCHEMA, + targetContent: z.string(), + }) + .strict(); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts new file mode 100644 index 00000000..7e8147b0 --- /dev/null +++ b/tests/findings/scorer.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { calculateCheckScore } from '../../src/scoring'; +import { Severity } from '../../src/evaluators/types'; +import { scoreCheck } from '../../src/findings/scorer'; +import type { RawViolation } from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + analysis: 'Issue', + message: 'Issue message', + fix: 'Fix', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +describe('scoreCheck', () => { + it('returns the same numeric result as calculateCheckScore for the same density', () => { + const verified = [violation(), violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + const direct = calculateCheckScore(verified, 100, { + strictness: 'standard', + promptSeverity: Severity.WARNING, + }); + + expect(result.score).toBe(direct.final_score); + expect(result.scoreText).toBe(`${direct.final_score.toFixed(1)}/10`); + expect(result.severity).toBe(direct.severity); + expect(result.findingCount).toBe(verified.length); + expect(result.findingCount).toBe(direct.violation_count); + }); + + it('is driven by the verified finding count, not a raw candidate count', () => { + // 1 verified violation over 100 words at standard strictness: + // density 1 -> 100 - 1*10 = 90 -> 9.0/10 + const one = scoreCheck({ + verifiedViolations: [violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(one.score).toBe(9.0); + expect(one.scoreText).toBe('9.0/10'); + + // 2 verified violations -> density 2 -> 100 - 2*10 = 80 -> 8.0/10 + const two = scoreCheck({ + verifiedViolations: [violation(), violation()], + wordCount: 100, + strictness: 'standard', + }); + expect(two.score).toBe(8.0); + expect(two.findingCount).toBe(2); + }); + + it('resolves error severity from the density score when score is low and prompt severity is error', () => { + // 20 violations over 100 words at standard strictness: + // density 20 -> 100 - 20*10 = -100 -> clamp 0 -> 0.0/10 -> prompt severity + const result = scoreCheck({ + verifiedViolations: Array.from({ length: 20 }, () => violation()), + wordCount: 100, + strictness: 'standard', + promptSeverity: Severity.ERROR, + }); + expect(result.score).toBe(0); + expect(result.severity).toBe(Severity.ERROR); + }); + + it('forwards strictness and prompt severity options to calculateCheckScore', () => { + const verified = [violation()]; + const result = scoreCheck({ + verifiedViolations: verified, + wordCount: 100, + strictness: 'strict', + }); + const direct = calculateCheckScore(verified, 100, { strictness: 'strict' }); + expect(result.score).toBe(direct.final_score); + }); +}); diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts new file mode 100644 index 00000000..46e0c93a --- /dev/null +++ b/tests/findings/severity.test.ts @@ -0,0 +1,73 @@ +import { describe, it, expect } from 'vitest'; +import { + buildRuleId, + resolveCriterionId, + resolveSeverity, +} from '../../src/findings/severity'; +import { EvaluationType, Severity } from '../../src/evaluators/types'; +import type { CheckResult } from '../../src/prompts/schema'; + +function makeCheckResult(severity: Severity): CheckResult { + return { + type: EvaluationType.CHECK, + final_score: 5, + percentage: 50, + violation_count: 1, + items: [], + severity, + message: 'Found 1 issue', + violations: [], + }; +} + +describe('resolveSeverity', () => { + it('returns the density-derived severity from the check score result', () => { + expect(resolveSeverity({ scored: makeCheckResult(Severity.WARNING) })).toBe( + Severity.WARNING, + ); + expect(resolveSeverity({ scored: makeCheckResult(Severity.ERROR) })).toBe( + Severity.ERROR, + ); + }); +}); + +describe('buildRuleId', () => { + it('builds Pack.Rule when no criterion is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern')).toBe('VectorLint.AiPattern'); + }); + + it('builds Pack.Rule.Criterion when a criterion id is given', () => { + expect(buildRuleId('VectorLint', 'AiPattern', 'Hedging')).toBe( + 'VectorLint.AiPattern.Hedging', + ); + }); + + it('ignores an empty criterion id', () => { + expect(buildRuleId('VectorLint', 'AiPattern', '')).toBe( + 'VectorLint.AiPattern', + ); + }); +}); + +describe('resolveCriterionId', () => { + const criteria = [ + { id: 'Hedging', name: 'Hedge words' }, + { id: 'Filler', name: 'Filler phrases' }, + ]; + + it('maps a criterion name to its declared id', () => { + expect(resolveCriterionId(criteria, 'Hedge words')).toBe('Hedging'); + }); + + it('returns undefined when the name does not match a criterion', () => { + expect(resolveCriterionId(criteria, 'Unknown')).toBeUndefined(); + }); + + it('returns undefined when no criterion name is provided', () => { + expect(resolveCriterionId(criteria, undefined)).toBeUndefined(); + }); + + it('returns undefined when no criteria are declared', () => { + expect(resolveCriterionId(undefined, 'Hedge words')).toBeUndefined(); + }); +}); diff --git a/tests/findings/types.test.ts b/tests/findings/types.test.ts new file mode 100644 index 00000000..4309ead6 --- /dev/null +++ b/tests/findings/types.test.ts @@ -0,0 +1,145 @@ +import { describe, it, expect } from 'vitest'; +import { + FINDING_PROCESSING_INPUT_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function validInput() { + return { + pack: 'VectorLint', + ruleId: 'AiPattern', + ruleSource: '/repo/presets/VectorLint/ai-pattern.md', + candidateFindings: [ + { + line: 1, + quoted_text: 'leverage synergies', + context_before: '', + context_after: '', + description: 'cliché phrase', + analysis: 'vague corporate cliché', + message: 'Avoid cliché phrase', + suggestion: 'Use concrete verbs', + fix: 'work together', + rule_quote: 'Avoid clichés', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + }, + ], + wordCount: 100, + promptMeta: { + severity: 'warning' as const, + strictness: 'standard' as const, + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + targetContent: 'leverage synergies everywhere', + }; +} + +describe('RAW_VIOLATION_SCHEMA', () => { + it('parses a representative candidate finding', () => { + const parsed = RAW_VIOLATION_SCHEMA.parse(validInput().candidateFindings[0]); + expect(parsed.analysis).toBe('vague corporate cliché'); + expect(parsed.checks?.evidence_exact).toBe(true); + }); + + it('requires analysis', () => { + expect(() => + RAW_VIOLATION_SCHEMA.parse({ message: 'no analysis field' }), + ).toThrow(); + }); +}); + +describe('PROMPT_META_FOR_FINDINGS_SCHEMA', () => { + it('rejects legacy evaluator/type/evaluateAs/target fields', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + severity: 'warning', + evaluator: 'base', + }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ type: 'judge' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ evaluateAs: 'document' }), + ).toThrow(); + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ target: { regex: 'x' } }), + ).toThrow(); + }); + + it('rejects rubric weight on criteria', () => { + expect(() => + PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ + criteria: [{ id: 'Hedging', name: 'Hedge words', weight: 2 }], + }), + ).toThrow(); + }); +}); + +describe('FINDING_PROCESSING_INPUT_SCHEMA', () => { + it('parses a representative supported finding-processing input', () => { + const parsed = FINDING_PROCESSING_INPUT_SCHEMA.parse(validInput()); + expect(parsed.pack).toBe('VectorLint'); + expect(parsed.candidateFindings).toHaveLength(1); + expect(parsed.promptMeta.criteria?.[0]?.id).toBe('Hedging'); + }); + + it('has no modelCall/mode/agent field and rejects them', () => { + for (const legacyKey of ['modelCall', 'mode', 'agent'] as const) { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), [legacyKey]: 'agent' }), + ).toThrow(); + } + }); + + it('rejects legacy judge/evaluator/rubric-shaped top-level input', () => { + // Legacy judge result shape: criteria array with scores at the top level. + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + pack: 'VectorLint', + ruleId: 'Tone', + ruleSource: '/x.md', + wordCount: 100, + targetContent: 'x', + promptMeta: {}, + evaluator: 'judge', + criteria: [{ name: 'Tone', score: 2, violations: [] }], + }), + ).toThrow(); + }); + + it('rejects a legacy evaluator field nested under promptMeta', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + promptMeta: { ...validInput().promptMeta, evaluator: 'base' }, + }), + ).toThrow(); + }); + + it('rejects malformed candidate findings', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ + ...validInput(), + candidateFindings: [{ message: 'no analysis field' }], + }), + ).toThrow(); + }); + + it('rejects an unknown top-level key', () => { + expect(() => + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), judge: true }), + ).toThrow(); + }); +}); From 20d42ff7d9441f3269d6fdea53f527ead80342b8 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 02:15:37 +0100 Subject: [PATCH 12/24] Implement shared finding processor contract - Add src/findings/processor.ts: processFindings runs the single pipeline (filter -> verify -> dedupe -> score -> severity -> diagnostics) and returns the Phase 2 ReviewResult from src/review/types.ts. - Verified finding count, not raw candidate count, drives the score; unanchored evidence becomes a warn diagnostic and emits no finding; hadOperationalErrors is true only for error-level diagnostics. - Add src/findings/index.ts barrel export. - Golden processor test asserts findings/score match the standard check path; cover diagnostics, filtering, verified-count fix, dedupe, rule-id attribution, and ReviewResult contract validation. --- src/findings/index.ts | 48 +++++ src/findings/processor.ts | 167 ++++++++++++++++ tests/findings/module-surface.test.ts | 27 +++ tests/findings/processor.test.ts | 278 ++++++++++++++++++++++++++ 4 files changed, 520 insertions(+) create mode 100644 src/findings/index.ts create mode 100644 src/findings/processor.ts create mode 100644 tests/findings/module-surface.test.ts create mode 100644 tests/findings/processor.test.ts diff --git a/src/findings/index.ts b/src/findings/index.ts new file mode 100644 index 00000000..095b878c --- /dev/null +++ b/src/findings/index.ts @@ -0,0 +1,48 @@ +/** + * Shared finding-processing layer (Phase 3). + * + * Transforms raw candidate violation findings into a formatter-ready + * `ReviewResult` through one pipeline: evidence verification, filtering, + * count/density scoring, severity resolution, and diagnostics. + * + * This module is independent of model call (`single` | `agent`) and contains + * no legacy `judge`, `evaluator`, rubric, or autonomous-loop output + * compatibility code. See + * docs/plans/2026-07-10-phase-3-share-result-projection.md. + */ +export { + FINDING_EVIDENCE_NOT_LOCATABLE, + verifyFindingEvidence, + type FindingEvidenceDiagnostic, + type FindingEvidenceInput, + type FindingEvidenceVerification, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; + +export { + buildRuleId, + resolveCriterionId, + resolveSeverity, + type SeverityInput, +} from './severity'; + +export { scoreCheck, type ScoreCheckOptions, type ScoredCheck } from './scorer'; + +export { + processFindings, +} from './processor'; + +export { + FINDING_PROCESSING_INPUT_SCHEMA, + FINDINGS_CRITERION_SCHEMA, + GATE_CHECKS_SCHEMA, + PROMPT_META_FOR_FINDINGS_SCHEMA, + RAW_VIOLATION_SCHEMA, + type FindingsCriterion, + type FindingProcessingInput, + type GateChecks, + type PromptMetaForFindings, + type RawViolation, + type RuleSeverity, + type Strictness, +} from './types'; diff --git a/src/findings/processor.ts b/src/findings/processor.ts new file mode 100644 index 00000000..45a1c7ae --- /dev/null +++ b/src/findings/processor.ts @@ -0,0 +1,167 @@ +import type { + ReviewDiagnostic, + ReviewFinding, + ReviewResult, + ReviewScore, +} from '../review/types'; +import { computeFilterDecision } from '../evaluators/violation-filter'; +import { + verifyFindingEvidence, + FINDING_EVIDENCE_NOT_LOCATABLE, + type VerifiedFindingCoords, +} from './finding-evidence-verifier'; +import { scoreCheck } from './scorer'; +import { buildRuleId, resolveCriterionId } from './severity'; +import type { FindingProcessingInput, RawViolation } from './types'; + +interface VerifiedEntry { + raw: RawViolation; + coords: VerifiedFindingCoords; +} + +/** + * The single finding-processing pipeline for objective Via Negativa violation + * checks (Phase 3). Transforms raw candidate findings into a formatter-ready + * {@link ReviewResult}. + * + * Order (audit lines 167-174): + * 1. filter candidates by the evidence gate (`computeFilterDecision`); + * 2. verify each surfaced finding's evidence (anchored quote or diagnostic); + * 3. deduplicate verified findings by `(quoted_text, line)`; + * 4. score by verified finding count/density; + * 5. resolve severity from the density score; + * 6. assemble findings, score, and diagnostics. + * + * The verified finding count — not the raw candidate count — drives the score + * (audit Finding #6). Unanchored evidence becomes a + * `finding-evidence-not-locatable` warn diagnostic and emits no finding. + * `hadOperationalErrors` is `true` only when a diagnostic is `error` level. + */ +export function processFindings(input: FindingProcessingInput): ReviewResult { + const surfaced = filterCandidates(input.candidateFindings); + const { verified, evidenceDiagnostics } = verifyAndDedupe( + surfaced, + input.targetContent, + ); + + const scored = scoreCheck({ + verifiedViolations: verified.map((entry) => entry.raw), + wordCount: input.wordCount, + ...(input.promptMeta.strictness !== undefined + ? { strictness: input.promptMeta.strictness } + : {}), + ...(input.promptMeta.severity !== undefined + ? { promptSeverity: input.promptMeta.severity } + : {}), + }); + + const severity = scored.severity; + const scoreRuleId = buildRuleId(input.pack, input.ruleId); + + const findings: ReviewFinding[] = verified.map(({ raw, coords }) => { + const criterionId = resolveCriterionId( + input.promptMeta.criteria, + raw.criterionName, + ); + const ruleId = buildRuleId(input.pack, input.ruleId, criterionId); + const message = (raw.message || raw.analysis || '').trim(); + return { + ruleId, + ruleSource: input.ruleSource, + severity, + message, + line: coords.line, + column: coords.column, + match: coords.match, + ...(raw.analysis ? { analysis: raw.analysis } : {}), + ...(raw.suggestion ? { suggestion: raw.suggestion } : {}), + ...(raw.fix ? { fix: raw.fix } : {}), + }; + }); + + const scores: ReviewScore[] = [ + { + ruleId: scoreRuleId, + score: scored.score, + scoreText: scored.scoreText, + severity, + findingCount: verified.length, + }, + ]; + + const diagnostics: ReviewDiagnostic[] = [...evidenceDiagnostics]; + const hadOperationalErrors = diagnostics.some( + (diagnostic) => diagnostic.level === 'error', + ); + + return { findings, scores, diagnostics, hadOperationalErrors }; +} + +/** + * Drops candidate findings that fail the evidence gate. Filtered candidates + * contribute neither findings nor diagnostics, matching the current standard + * orchestrator behavior. + */ +function filterCandidates(candidates: readonly RawViolation[]): RawViolation[] { + const surfaced: RawViolation[] = []; + for (const candidate of candidates) { + const decision = computeFilterDecision(candidate); + if (decision.surface) { + surfaced.push(candidate); + } + } + return surfaced; +} + +/** + * Verifies each surfaced finding's evidence and deduplicates verified findings + * by `(quoted_text, line)`. Unanchored findings become warn diagnostics and are + * not emitted. + */ +function verifyAndDedupe( + surfaced: readonly RawViolation[], + targetContent: string, +): { verified: VerifiedEntry[]; evidenceDiagnostics: ReviewDiagnostic[] } { + const verified: VerifiedEntry[] = []; + const evidenceDiagnostics: ReviewDiagnostic[] = []; + const seen = new Set(); + + for (const candidate of surfaced) { + const verification = verifyFindingEvidence(targetContent, { + quoted_text: candidate.quoted_text || '', + ...(candidate.context_before !== undefined + ? { context_before: candidate.context_before } + : {}), + ...(candidate.context_after !== undefined + ? { context_after: candidate.context_after } + : {}), + ...(candidate.line !== undefined ? { line: candidate.line } : {}), + }); + + if (!verification.verified || !verification.finding) { + evidenceDiagnostics.push({ + level: 'warn', + code: FINDING_EVIDENCE_NOT_LOCATABLE, + message: + verification.diagnostic?.message ?? + 'Could not locate finding evidence in target content.', + }); + continue; + } + + const coords = verification.finding; + const dedupeKey = candidate.quoted_text + ? `${candidate.quoted_text}:${coords.line}` + : null; + if (dedupeKey) { + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + } + + verified.push({ raw: candidate, coords }); + } + + return { verified, evidenceDiagnostics }; +} diff --git a/tests/findings/module-surface.test.ts b/tests/findings/module-surface.test.ts new file mode 100644 index 00000000..a0e5c481 --- /dev/null +++ b/tests/findings/module-surface.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import * as findings from '../../src/findings'; + +describe('src/findings public surface', () => { + it('exports the processing entry point and helpers', () => { + expect(typeof findings.processFindings).toBe('function'); + expect(typeof findings.verifyFindingEvidence).toBe('function'); + expect(typeof findings.resolveSeverity).toBe('function'); + expect(typeof findings.buildRuleId).toBe('function'); + expect(typeof findings.resolveCriterionId).toBe('function'); + expect(typeof findings.scoreCheck).toBe('function'); + }); + + it('exports the stable diagnostic code constant', () => { + expect(findings.FINDING_EVIDENCE_NOT_LOCATABLE).toBe( + 'finding-evidence-not-locatable', + ); + }); + + it('exports the Zod boundary schemas', () => { + expect(findings.FINDING_PROCESSING_INPUT_SCHEMA).toBeDefined(); + expect(findings.RAW_VIOLATION_SCHEMA).toBeDefined(); + expect(findings.GATE_CHECKS_SCHEMA).toBeDefined(); + expect(findings.FINDINGS_CRITERION_SCHEMA).toBeDefined(); + expect(findings.PROMPT_META_FOR_FINDINGS_SCHEMA).toBeDefined(); + }); +}); diff --git a/tests/findings/processor.test.ts b/tests/findings/processor.test.ts new file mode 100644 index 00000000..90f9641d --- /dev/null +++ b/tests/findings/processor.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { processFindings } from '../../src/findings/processor'; +import { FINDING_EVIDENCE_NOT_LOCATABLE } from '../../src/findings/finding-evidence-verifier'; +import { REVIEW_RESULT_SCHEMA } from '../../src/review'; +import type { + FindingProcessingInput, + RawViolation, +} from '../../src/findings/types'; + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +function violation(overrides: Partial = {}): RawViolation { + return { + line: 1, + quoted_text: 'Alpha text', + context_before: '', + context_after: '', + analysis: 'Issue 1', + message: 'Issue 1', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + rule_quote: 'Rule quote', + confidence: 0.9, + checks: FULLY_SUPPORTED_CHECKS, + ...overrides, + }; +} + +function baseInput( + candidateFindings: RawViolation[], + overrides: Partial = {}, +): FindingProcessingInput { + return { + pack: 'TestPack', + ruleId: 'CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + candidateFindings, + wordCount: 100, + promptMeta: { severity: 'warning', strictness: 'standard' }, + targetContent: 'Alpha text\nBeta text\n', + ...overrides, + }; +} + +describe('processFindings', () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + // Default threshold (0.75) unless a test opts in. + delete process.env.CONFIDENCE_THRESHOLD; + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + }); + + describe('golden objective-check output', () => { + beforeEach(() => { + // Surface every gate-passing candidate regardless of confidence. + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('produces byte-for-byte findings/score matching the standard orchestrator path', () => { + const input = baseInput([ + violation(), + violation({ + line: 2, + quoted_text: 'Beta text', + analysis: 'Issue 2', + message: 'Issue 2', + suggestion: 'Suggestion 2', + fix: 'Fix 2', + }), + ]); + + const result = processFindings(input); + + // Score: density 2/100 at standard strictness (10) -> 100 - 20 -> 8.0/10. + expect(result.scores).toEqual([ + { + ruleId: 'TestPack.CheckPrompt', + score: 8.0, + scoreText: '8.0/10', + severity: 'warning', + findingCount: 2, + }, + ]); + + // Findings: Pack.Rule (check violations carry no criterion name), + // anchored lines/columns/match. + expect(result.findings).toHaveLength(2); + expect(result.findings[0]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + ruleSource: '/repo/prompts/CheckPrompt.md', + severity: 'warning', + message: 'Issue 1', + line: 1, + column: 1, + match: 'Alpha text', + suggestion: 'Suggestion 1', + fix: 'Fix 1', + }); + expect(result.findings[1]).toMatchObject({ + ruleId: 'TestPack.CheckPrompt', + severity: 'warning', + message: 'Issue 2', + line: 2, + column: 1, + match: 'Beta text', + }); + + expect(result.diagnostics).toEqual([]); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('returns a ReviewResult that validates against the review contract', () => { + const result = processFindings(baseInput([violation()])); + expect(() => REVIEW_RESULT_SCHEMA.parse(result)).not.toThrow(); + }); + }); + + describe('evidence verification (audit Finding #6)', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('turns unanchored evidence into a warn diagnostic and emits no finding', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'this quote is not in the content' }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toHaveLength(1); + expect(result.diagnostics[0]?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); + expect(result.diagnostics[0]?.level).toBe('warn'); + // Score reflects 0 verified findings -> perfect 10.0/10. + expect(result.scores[0]?.score).toBe(10.0); + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('counts only verified findings toward the score, not raw candidate count', () => { + // One anchored + one unanchored candidate. Verified count = 1. + // Density 1/100 -> 100 - 10 -> 9.0/10 (the intentional fix). + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text' }), + violation({ quoted_text: 'nowhere to be found' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.score).toBe(9.0); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.diagnostics).toHaveLength(1); + }); + + it('does not set hadOperationalErrors for warn-level evidence diagnostics', () => { + const result = processFindings( + baseInput([violation({ quoted_text: 'missing' })]), + ); + expect(result.diagnostics[0]?.level).toBe('warn'); + expect(result.hadOperationalErrors).toBe(false); + }); + + it('deduplicates verified findings by quoted_text and line', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', line: 1 }), + violation({ quoted_text: 'Alpha text', line: 1, message: 'dup' }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + }); + }); + + describe('candidate filtering', () => { + it('drops candidates that fail computeFilterDecision without a diagnostic', () => { + // Default threshold 0.75: confidence 0.2 is filtered out. + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.2 }), + ]), + ); + + expect(result.findings).toEqual([]); + expect(result.diagnostics).toEqual([]); + // No verified findings -> 10.0/10. + expect(result.scores[0]?.findingCount).toBe(0); + }); + + it('filtered candidates do not contribute to the score count', () => { + const result = processFindings( + baseInput([ + violation({ quoted_text: 'Alpha text', confidence: 0.9 }), + violation({ quoted_text: 'Beta text', line: 2, confidence: 0.2 }), + ]), + ); + + expect(result.findings).toHaveLength(1); + expect(result.scores[0]?.findingCount).toBe(1); + expect(result.scores[0]?.score).toBe(9.0); + }); + }); + + describe('rule id building', () => { + beforeEach(() => { + process.env.CONFIDENCE_THRESHOLD = '0.0'; + }); + + it('attributes findings to Pack.Rule when violations carry no criterion name', () => { + const result = processFindings(baseInput([violation({ quoted_text: 'Alpha text' })])); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt'); + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + + it('attributes findings to Pack.Rule.Criterion when a violation names a declared criterion', () => { + const result = processFindings( + baseInput( + [ + violation({ + quoted_text: 'Alpha text', + criterionName: 'Hedge words', + }), + ], + { + promptMeta: { + severity: 'warning', + strictness: 'standard', + criteria: [{ id: 'Hedging', name: 'Hedge words' }], + }, + }, + ), + ); + expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt.Hedging'); + // The score stays at Pack.Rule (no criterion), matching the orchestrator. + expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + }); + }); + + describe('severity resolution', () => { + it('resolves error severity from the density score and stamps it on findings', () => { + // 20 verified violations over 100 words -> 0.0/10 -> prompt severity error. + const result = processFindings( + baseInput( + Array.from({ length: 20 }, (_, i) => + violation({ + quoted_text: `Alpha text${i}`, + analysis: `Issue ${i}`, + message: `Issue ${i}`, + }), + ), + { + targetContent: Array.from({ length: 20 }, (_, i) => `Alpha text${i}`).join('\n') + '\n', + promptMeta: { severity: 'error', strictness: 'standard' }, + }, + ), + ); + + expect(result.scores[0]?.severity).toBe('error'); + expect(result.findings.every((f) => f.severity === 'error')).toBe(true); + }); + }); +}); From 671da9cd5e1cfae9863970274e338c541fc63893 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 03:24:02 +0100 Subject: [PATCH 13/24] Route standard check evaluation through shared processor - Build FindingProcessingInput and call processFindings in the standard objective-check branch of routePromptResult - Feed returned findings/scores/diagnostics through the existing line/json/rdjson/vale sinks and quality-score output - Remove inline filter/score/severity/quote-location logic from the check path; computeFilterDecision kept only for the debug-run artifact - Verified finding count now drives counts and score (audit Finding #6); unanchored quotes become warn diagnostics and no longer flag operational errors - Add orchestrator regression tests covering finding/score output, JSON/Vale shape, and exit behavior for supported objective-check reviews --- src/cli/orchestrator.ts | 120 ++++--- tests/orchestrator-check-processor.test.ts | 356 +++++++++++++++++++++ 2 files changed, 413 insertions(+), 63 deletions(-) create mode 100644 tests/orchestrator-check-processor.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f8df367a..a8e24436 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -12,6 +12,7 @@ import { printFileHeader, printIssueRow, printEvaluationSummaries, type Evaluati import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; +import { processFindings } from '../findings'; import { createEvaluator } from '../evaluators/index'; import { Type, Severity } from '../evaluators/types'; import { USER_INSTRUCTION_FILENAME } from '../config/constants'; @@ -616,79 +617,72 @@ function routePromptResult( let promptErrors = 0; let promptWarnings = 0; - // Handle Check Result + // Handle Check Result — routed through the shared finding processor + // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, + // filters, deduplicates, scores, and resolves severity; the orchestrator + // only feeds the returned ReviewResult to the existing formatter sinks. if (!isJudgeResult(result)) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - - // Score calculated from surfaced violations only — matches what user sees - const scored = calculateCheckScore( - surfacedViolations, - result.word_count, - { - strictness: promptFile.meta.strictness, - promptSeverity: promptFile.meta.severity, - } - ); - const severity = scored.severity; - // Group violations by criterionName - const violationsByCriterion = new Map< - string | undefined, - typeof surfacedViolations - >(); - for (const v of surfacedViolations) { - const criterionName = v.criterionName; - if (!violationsByCriterion.has(criterionName)) { - violationsByCriterion.set(criterionName, []); - } - violationsByCriterion.get(criterionName)!.push(v); - } - - // Report violations grouped by criterion - let totalErrors = 0; - let totalWarnings = 0; + const reviewResult = processFindings({ + pack: promptFile.pack, + ruleId: promptId, + ruleSource: promptFile.fullPath, + candidateFindings: result.violations, + wordCount: result.word_count, + promptMeta: { + ...(meta.severity !== undefined ? { severity: meta.severity } : {}), + ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), + ...(meta.criteria ? { criteria: meta.criteria } : {}), + }, + targetContent: content, + }); - for (const [criterionName, violations] of violationsByCriterion) { - // Find criterion ID from meta - let criterionId: string | undefined; - if (criterionName && meta.criteria) { - const criterion = meta.criteria.find(c => c.name === criterionName); - criterionId = criterion?.id; - } + // processFindings always returns exactly one score entry (processor contract). + const ruleScore = reviewResult.scores[0]!; + const severity = + ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - const ruleName = buildRuleName(promptFile.pack, promptId, criterionId); - - if (violations.length > 0) { - const violationResult = locateAndReportViolations({ - violations, - content, - relFile, - severity, - ruleName, - scoreText: '', - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; + // Report only verified findings through the existing line/json/rdjson/vale sink. + for (const finding of reviewResult.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity, + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); + } - if (severity === Severity.ERROR) { - totalErrors += violations.length; - } else { - totalWarnings += violations.length; - } + // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, + // consistent with prior operational reporting. They are warn-level and do not + // flag the run as operationally failed (audit Finding #6). + if (verbose) { + for (const diagnostic of reviewResult.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } } - // Create scoreEntry for Quality Scores display + // Verified finding count drives the counts (audit Finding #6). + const findingCount = reviewResult.findings.length; + const totalErrors = severity === Severity.ERROR ? findingCount : 0; + const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; + const scoreEntry: EvaluationSummary = { - id: buildRuleName(promptFile.pack, promptId, undefined), - scoreText: `${scored.final_score.toFixed(1)}/10`, - score: scored.final_score, + id: ruleScore.ruleId, + scoreText: ruleScore.scoreText, + score: ruleScore.score, }; if (debugJson) { + const { decisions, surfacedViolations } = getViolationFilterResults( + result.violations + ); const runId = randomUUID(); const model = getModelInfoFromEnv(); @@ -721,7 +715,7 @@ function routePromptResult( return { errors: totalErrors, warnings: totalWarnings, - hadOperationalErrors, + hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, scoreEntries: [scoreEntry], }; diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-check-processor.test.ts new file mode 100644 index 00000000..eaaa1529 --- /dev/null +++ b/tests/orchestrator-check-processor.test.ts @@ -0,0 +1,356 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { Result } from "../src/output/json-formatter"; +import type { ValeOutput } from "../src/schemas/vale-responses"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; + +const { EVALUATE_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), +})); + +type CheckViolation = RawCheckResult["violations"][number]; + +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +} as const; + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +/** + * Builds a check violation that passes the confidence gate by default. Real + * model output always carries `message`; the helper sets it explicitly so the + * test reflects production behavior rather than the analysis fallback. + */ +function makeCheckViolation( + overrides: Partial = {} +): CheckViolation { + return { + line: 1, + analysis: "Issue 1", + message: "Issue 1", + suggestion: "Suggestion 1", + fix: "Fix 1", + quoted_text: "Alpha text", + context_before: "", + context_after: "", + rule_quote: "Rule quote", + checks: FULLY_SUPPORTED_CHECKS, + confidence: 0.9, + ...overrides, + }; +} + +function makeCheckResult(params: { + violations: CheckViolation[]; + wordCount?: number; +}): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: params.violations, + word_count: params.wordCount ?? 100, + }; +} + +describe("standard check evaluation via the shared finding processor", () => { + const originalThreshold = process.env.CONFIDENCE_THRESHOLD; + + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + delete process.env.CONFIDENCE_THRESHOLD; + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + if (originalThreshold === undefined) { + delete process.env.CONFIDENCE_THRESHOLD; + } else { + process.env.CONFIDENCE_THRESHOLD = originalThreshold; + } + vi.restoreAllMocks(); + }); + + it("reports fully locatable findings, score, and counts unchanged", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckPrompt", + name: "Check Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First issue", analysis: "First issue" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second issue", + analysis: "Second issue", + suggestion: "Suggestion 2", + fix: "Fix 2", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. + expect(run.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("8.0/10"); + // The line summary prints the last segment of the Pack.Rule score id. + expect(scoreLine).toContain("CheckPrompt"); + }); + + it("counts only verified findings and excludes unanchored quotes from the score", async () => { + // One anchored + one unanchored quote. Both pass the confidence gate, so + // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CountingFixPrompt", + name: "Counting Fix Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "this quote is not anywhere in the content", + message: "Ghost issue", + analysis: "Ghost issue", + }), + ], + wordCount: 100, + }) + ); + + const logCalls: string[] = []; + vi.spyOn(console, "log").mockImplementation((...args) => { + logCalls.push(args.map(String).join(" ")); + }); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a + // finding; it contributes neither to the count nor the score. + expect(run.totalWarnings).toBe(1); + expect(run.hadOperationalErrors).toBe(false); + + const scoreLine = logCalls.find((l) => l.includes("/10")); + expect(scoreLine).toBeDefined(); + expect(scoreLine).toContain("9.0/10"); + }); + + it("does not flag severity errors when no finding can be verified", async () => { + // severity=error, but the only quote cannot anchor -> 0 verified findings -> + // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorNoVerifiedPrompt", + name: "Error No Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ + line: 9, + quoted_text: "nowhere to be found", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(0); + expect(run.hadSeverityErrors).toBe(false); + expect(run.hadOperationalErrors).toBe(false); + }); + + it("flags severity errors when verified error-severity findings exist", async () => { + // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> + // prompt severity error applies. + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "ErrorVerifiedPrompt", + name: "Error Verified Prompt", + type: "check", + severity: Severity.ERROR, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [makeCheckViolation({ quoted_text: "Alpha text" })], + wordCount: 10, + }) + ); + + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(run.totalErrors).toBe(1); + expect(run.hadSeverityErrors).toBe(true); + }); + + it("emits verified findings with anchored location through the JSON sink", async () => { + const targetFile = createTempFile("Alpha text\nBeta text\n"); + const prompt = createPrompt({ + id: "CheckJsonPrompt", + name: "Check JSON Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ message: "First", analysis: "First" }), + makeCheckViolation({ + line: 2, + quoted_text: "Beta text", + message: "Second", + analysis: "Second", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.Json, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as Result; + const issues = Object.values(parsed.files).flatMap((file) => file.issues); + + expect(issues).toHaveLength(2); + expect(issues[0]).toMatchObject({ + line: 1, + column: 1, + severity: Severity.WARNING, + message: "First", + rule: "TestPack.CheckJsonPrompt", + match: "Alpha text", + }); + expect(issues[1]).toMatchObject({ + line: 2, + message: "Second", + match: "Beta text", + }); + }); + + it("omits unanchored quotes from the Vale JSON output", async () => { + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CheckValePrompt", + name: "Check Vale Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + violations: [ + makeCheckViolation({ quoted_text: "Alpha text" }), + makeCheckViolation({ + line: 9, + quoted_text: "missing quote", + message: "Ghost", + analysis: "Ghost", + }), + ], + wordCount: 100, + }) + ); + + await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.ValeJson, + }); + + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as ValeOutput; + const issues = Object.values(parsed).flat(); + + expect(issues).toHaveLength(1); + expect(issues[0]).toMatchObject({ + Check: "TestPack.CheckValePrompt", + Line: 1, + Match: "Alpha text", + Severity: "warning", + }); + }); +}); From 9fe6440c7cc768ec0da72cb2a7fd2d73330b51d5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:36 +0100 Subject: [PATCH 14/24] Reject judge prompt type at the loader boundary - PROMPT_META_SCHEMA now rejects `type: judge` and its deprecated alias `subjective` via a superRefine, so judge-typed prompts fail to load with a descriptive warning instead of reaching evaluation. - superRefine (not refine) keeps the inferred union including "judge" so the legacy BaseEvaluator type-checks unchanged until Phase 4 deletes that path. - Add loader tests covering judge/subjective rejection and that check still loads; subjective and semi-objective are documented as deprecated aliases. --- src/schemas/prompt-schemas.ts | 26 ++++++++++--- tests/prompt-loader-validation.test.ts | 51 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index f8689645..2dadd85e 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -27,12 +27,15 @@ export const PROMPT_CRITERION_SCHEMA = z.object({ * - 'technical-accuracy': specialized evaluator with claim extraction + search * * Evaluation type: - * - 'judge': 1-4 scores per criterion, normalized to 1-10 * - 'check': density-based scoring (errors per 100 words) + * - 'semi-objective': deprecated alias for 'check' * - * Deprecated aliases (still supported): - * - 'subjective' → 'judge' - * - 'semi-objective' → 'check' + * 'judge' and its deprecated alias 'subjective' are no longer supported + * (Phase 3): subjective rubric scoring is not a future-facing review type. + * The enum still admits them so the Zod error is descriptive and the inferred + * type stays compatible with the legacy BaseEvaluator until Phase 4 deletes + * that path; a superRefine rejects them at this boundary so judge-typed prompts + * fail to load. * * Strictness factor for check scoring: * - Determines penalty weight per 1% error density. @@ -44,11 +47,24 @@ export const PROMPT_META_SCHEMA = z.object({ type: z .enum(["judge", "check", "subjective", "semi-objective"]) .transform((val) => { - // Map deprecated values to new canonical values + // Map deprecated values to their canonical forms. if (val === "subjective") return "judge" as const; if (val === "semi-objective") return "check" as const; return val; }) + .superRefine((val, ctx) => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). + // superRefine (not refine) so the inferred union still includes 'judge' + // and stays compatible with the legacy BaseEvaluator until Phase 4 deletes + // that path; the value is rejected at this boundary regardless. + if (val === "judge") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "judge evaluation is no longer supported; use check (Via Negativa) rules", + }); + } + }) .optional(), id: z.string(), name: z.string(), diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index 094ad7cb..fc34cc1d 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -130,4 +130,55 @@ Check content.`); expect(warnings[0]).toContain('test.md'); }); }); + + describe('Judge boundary rejection (Phase 3)', () => { + it('rejects judge-typed prompts at the loader boundary', () => { + createPrompt('judge.md', `--- +evaluator: base +id: Judge +name: Judge Prompt +type: judge +criteria: + - name: Clarity + id: Clarity +--- +Judge content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('judge.md'); + expect(warnings[0]).toMatch(/judge/i); + }); + + it('rejects subjective-typed prompts (deprecated judge alias)', () => { + createPrompt('subjective.md', `--- +evaluator: base +id: Subjective +name: Subjective Prompt +type: subjective +--- +Subjective content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(prompts).toHaveLength(0); + expect(warnings.length).toBeGreaterThan(0); + expect(warnings[0]).toContain('subjective.md'); + }); + + it('still loads check-typed prompts', () => { + createPrompt('check.md', `--- +evaluator: base +id: Check +name: Check Prompt +type: check +--- +Check content.`); + + const { prompts, warnings } = loadRules(tmpDir); + expect(warnings).toHaveLength(0); + expect(prompts).toHaveLength(1); + expect(prompts[0].meta.type).toBe('check'); + }); + }); }); From d36d06dd139439667ac765b4adfe1e1162a76e42 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 07:30:54 +0100 Subject: [PATCH 15/24] Remove legacy judge review path - Drop the judge branch of routePromptResult and the now-dead criterion extraction/reporting helpers (extractAndReportCriterion, validateCriteriaCompleteness, validateScores, locateAndReportViolations, buildRuleName); subjective rubric scoring is not a future-facing review type. - A JudgeResult that still reaches the orchestrator is refused explicitly (operational error, no findings) rather than misprojected as a check result; no adapter is added for old judge metadata. - Standard objective-check orchestration still routes through processFindings unchanged. - Replace the judge-filtering orchestrator test with a judge-rejection test. --- src/cli/orchestrator.ts | 545 +-------------------------- tests/orchestrator-filtering.test.ts | 41 +- 2 files changed, 28 insertions(+), 558 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index a8e24436..4d09e917 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -6,10 +6,9 @@ import * as os from 'os'; import type { PromptFile } from '../prompts/prompt-loader'; import { ScanPathResolver } from '../boundaries/scan-path-resolver'; import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter'; -import { JsonFormatter, type Issue, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { checkTarget } from '../prompts/target'; import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { processFindings } from '../findings'; @@ -21,8 +20,7 @@ import { runAgentExecutor, type AgentExecutorResult, type AgentFinding } from '. import { AgentProgressReporter, shouldEmitAgentProgress } from '../agent/progress'; import type { EvaluationOptions, EvaluationResult, ErrorTrackingResult, - ReportIssueParams, ProcessViolationsParams, - ProcessCriterionParams, ProcessCriterionResult, ValidationParams, ProcessPromptResultParams, + ReportIssueParams, ProcessPromptResultParams, RunPromptEvaluationParams, RunPromptEvaluationResult, EvaluateFileParams, EvaluateFileResult, RunPromptEvaluationResultSuccess } from './types'; @@ -33,7 +31,6 @@ import { import { calculateCheckScore } from '../scoring'; import { countWords } from '../chunking/utils'; import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; -import { locateQuotedText } from "../output/location"; import { computeFilterDecision, type FilterDecision, @@ -64,23 +61,6 @@ function getModelInfoFromEnv(): { provider?: string; name?: string; tag?: string } -/* - * Constructs a hierarchical rule name following the pattern: - * - With criterion: PackName.RuleId.CriterionId - * - Without criterion: PackName.RuleId - */ -function buildRuleName( - packName: string, - ruleId: string, - criterionId: string | undefined -): string { - const parts = [packName, ruleId]; - if (criterionId) { - parts.push(criterionId); - } - return parts.join('.'); -} - /* * Generic concurrency runner that executes workers in parallel up to a specified limit. * Preserves result order matching input order. @@ -174,119 +154,6 @@ function reportIssue(params: ReportIssueParams): void { } } -/* - * Locates and reports each violation using pre/post evidence markers. - * If location matching fails (missing markers, content mismatch), logs warning - * and continues processing. Returns hadOperationalErrors=true if any violations - * couldn't be located, signaling text matching issues vs. content quality issues. - */ -function locateAndReportViolations(params: ProcessViolationsParams): { - hadOperationalErrors: boolean; -} { - const { - violations, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose, - } = params; - - let hadOperationalErrors = false; - - // Locate all violations and filter out those that can't be verified - // Then de-duplicate by (quoted_text, line) - const seen = new Set(); - const verifiedViolations: Array<{ - v: (typeof violations)[0]; - line: number; - column: number; - matchedText: string; - rowSummary: string; - }> = []; - - for (const v of violations) { - if (!v) continue; - - const rowSummary = (v.message || "").trim(); - - try { - const locWithMatch = locateQuotedText( - content, - { - quoted_text: v.quoted_text || "", - context_before: v.context_before || "", - context_after: v.context_after || "", - }, - 80, - v.line - ); - - if (!locWithMatch) { - // Can't verify this quote exists - skip it entirely - if (verbose) { - console.warn( - `[vectorlint] Skipping unverifiable quote: "${v.quoted_text}"` - ); - } - hadOperationalErrors = true; - continue; - } - - const line = locWithMatch.line; - const column = locWithMatch.column; - const matchedText = locWithMatch.match || ""; - - // De-duplicate by (quoted_text, line) - skip if quoted_text is empty - const dedupeKey = v.quoted_text ? `${v.quoted_text}:${line}` : null; - if (dedupeKey && seen.has(dedupeKey)) { - continue; // Skip duplicate - } - if (dedupeKey) { - seen.add(dedupeKey); - } - - verifiedViolations.push({ v, line, column, matchedText, rowSummary }); - } catch (e: unknown) { - const err = handleUnknownError(e, "Locating evidence"); - if (verbose) { - console.warn(`[vectorlint] Error locating evidence: ${err.message}`); - } - hadOperationalErrors = true; - } - } - - // Report only verified, unique violations - for (const { - v, - line, - column, - matchedText, - rowSummary, - } of verifiedViolations) { - reportIssue({ - file: relFile, - line, - column, - severity, - summary: rowSummary, - ruleName, - outputFormat, - jsonFormatter, - ...(v.analysis !== undefined && { analysis: v.analysis }), - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), - ...(v.fix !== undefined && { fix: v.fix }), - scoreText, - match: matchedText, - }); - } - - return { hadOperationalErrors }; -} - function getViolationFilterResults< TViolation extends Parameters[0] >( @@ -304,297 +171,10 @@ function getViolationFilterResults< } /* - * Extracts pre-calculated scores from a subjective evaluation criterion and reports surfaced violations. - * Violations that do not pass computeFilterDecision are not reported. - * Returns error/warning counts, score entry for Quality Scores, and score components for JSON. - */ -function extractAndReportCriterion( - params: ProcessCriterionParams -): ProcessCriterionResult { - const { - exp, - result, - content, - relFile, - packName, - promptId, - meta, - outputFormat, - jsonFormatter, - verbose, - } = params; - let hadOperationalErrors = false; - let hadSeverityErrors = false; - - const nameKey = String(exp.name || exp.id || ""); - const criterionId = exp.id - ? String(exp.id) - : exp.name - ? String(exp.name) - .replace(/[^A-Za-z0-9]+/g, " ") - .split(" ") - .filter(Boolean) - .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) - .join("") - : ""; - const ruleName = buildRuleName(packName, promptId, criterionId); - - const weightNum = exp.weight || 1; - const maxScore = weightNum; - - // Target gating (deterministic precheck) - const metaTargetSpec = meta.target; - const expTargetSpec = exp.target; - const targetCheck = checkTarget(content, metaTargetSpec, expTargetSpec); - const missingTarget = targetCheck.missing; - - if (missingTarget) { - hadSeverityErrors = true; - const summary = "target not found"; - const suggestion = - targetCheck.suggestion || - expTargetSpec?.suggestion || - metaTargetSpec?.suggestion || - "Add the required target section."; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity: Severity.ERROR, - summary, - ruleName, - outputFormat, - jsonFormatter, - suggestion, - scoreText: "nil", - match: "", - }); - return { - errors: 1, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "0.0/10", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const got = result.criteria.find( - (c) => c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) { - return { - errors: 0, - warnings: 0, - userScore: 0, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText: "-", score: 0.0 }, - scoreComponent: { - criterion: nameKey, - rawScore: 0, - maxScore: 4, - weightedScore: 0, - weightedMaxScore: weightNum, - normalizedScore: 0, - normalizedMaxScore: 10, - }, - }; - } - - const score = Number(got.score); - - // Use pre-calculated values from evaluator - const rawWeighted = got.weighted_points; - const normalizedScore = got.normalized_score; - const userScore = rawWeighted; - const violations = got.violations; - const { surfacedViolations } = getViolationFilterResults(violations); - - // Display normalized score (1-10) in CLI output - const scoreText = `${normalizedScore.toFixed(1)}/10`; - - // Determine severity based on violations - // If there are violations, use evaluator's scoring to determine severity - // Score <= 1 = error, score = 2 = warning, score > 2 = no severity needed (but we still create scoreEntry) - let errors = 0; - let warnings = 0; - let severity: Severity | undefined; - - if (surfacedViolations.length > 0) { - // Determine severity from score for violations - if (score <= 1) { - severity = Severity.ERROR; - hadSeverityErrors = true; - errors = surfacedViolations.length; - } else if (score === 2) { - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } else { - // Score > 2 but has violations - this is informational - // Use WARNING as default for informational violations - severity = Severity.WARNING; - warnings = surfacedViolations.length; - } - - // Report surfaced violations only - const violationResult = locateAndReportViolations({ - violations: surfacedViolations as Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - analysis?: string; - suggestion?: string; - }>, - content, - relFile, - severity, - ruleName, - scoreText, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - hadOperationalErrors = - hadOperationalErrors || violationResult.hadOperationalErrors; - } else if (score <= 2) { - // No violations but low score - report with summary - severity = score <= 1 ? Severity.ERROR : Severity.WARNING; - if (severity === Severity.ERROR) { - hadSeverityErrors = true; - errors = 1; - } else { - warnings = 1; - } - - const sum = got.summary.trim(); - const words = sum.split(/\s+/).filter(Boolean); - const limited = words.slice(0, 15).join(" "); - const summaryText = limited || "No findings"; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: summaryText, - ruleName, - outputFormat, - jsonFormatter, - scoreText, - match: "", - }); - } - - return { - errors, - warnings, - userScore, - maxScore, - hadOperationalErrors, - hadSeverityErrors, - scoreEntry: { id: ruleName, scoreText, score: normalizedScore }, - scoreComponent: { - criterion: nameKey, - rawScore: score, - maxScore: 4, - weightedScore: rawWeighted, - weightedMaxScore: weightNum, - normalizedScore: normalizedScore, - normalizedMaxScore: 10, - }, - }; -} - -/* - * Validates that all expected criteria are present in the result. - */ -function validateCriteriaCompleteness(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - const expectedNames = new Set( - (meta.criteria || []).map((c) => String(c.name || c.id || "")) - ); - const returnedNames = new Set( - result.criteria.map((c: { name: string }) => c.name) - ); - - // Create normalized maps for case-insensitive lookup - const expectedNormalized = new Set(); - const expectedOriginalMap = new Map(); - for (const name of expectedNames) { - const norm = name.toLowerCase(); - expectedNormalized.add(norm); - expectedOriginalMap.set(norm, name); - } - - const returnedNormalized = new Set(); - for (const name of returnedNames) { - returnedNormalized.add(name.toLowerCase()); - } - - for (const norm of expectedNormalized) { - if (!returnedNormalized.has(norm)) { - console.error( - `Missing criterion in model output: ${expectedOriginalMap.get(norm)}` - ); - hadErrors = true; - } - } - - for (const name of returnedNames) { - if (!expectedNormalized.has(name.toLowerCase())) { - console.warn( - `[vectorlint] Extra criterion returned by model (ignored): ${name}` - ); - } - } - - return hadErrors; -} - -/* - * Validates that all criterion scores are within valid range. - */ -function validateScores(params: ValidationParams): boolean { - const { meta, result } = params; - let hadErrors = false; - - for (const exp of meta.criteria || []) { - const nameKey = String(exp.name || exp.id || ""); - const got = result.criteria.find( - (c) => - c.name === nameKey || c.name.toLowerCase() === nameKey.toLowerCase() - ); - if (!got) continue; - - const score = Number(got.score); - if (!Number.isFinite(score) || score < 0 || score > 4) { - console.error(`Invalid score for ${nameKey}: ${score}`); - hadErrors = true; - } - } - - return hadErrors; -} - -/* - * Routes evaluation results through check or judge processing paths. - * Check: Reports violations, creates scoreEntry using final_score. - * Judge: Iterates through criteria, validates scores, creates scoreEntry per criterion. - * Both paths generate scoreEntries for Quality Scores display. + * Routes an evaluation result through the shared finding processor. + * Check results are verified, filtered, scored, and reported; judge/rubric + * results are rejected (Phase 3) because subjective scoring is not a + * future-facing review type. */ function routePromptResult( params: ProcessPromptResultParams @@ -612,11 +192,6 @@ function routePromptResult( const meta = promptFile.meta; const promptId = (meta.id || "").toString(); - let hadOperationalErrors = false; - let hadSeverityErrors = false; - let promptErrors = 0; - let promptWarnings = 0; - // Handle Check Result — routed through the shared finding processor // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, // filters, deduplicates, scores, and resolves severity; the orchestrator @@ -721,105 +296,21 @@ function routePromptResult( }; } - // Handle Judge Result - // Validate criterion completeness and scores - hadOperationalErrors = - validateCriteriaCompleteness({ meta, result }) || hadOperationalErrors; - hadOperationalErrors = - validateScores({ meta, result }) || hadOperationalErrors; - - // Reset promptErrors and promptWarnings for subjective results - promptErrors = 0; - promptWarnings = 0; - const criterionScores: EvaluationSummary[] = []; - const scoreComponents: ScoreComponent[] = []; - - // Iterate through each criterion - for (const exp of meta.criteria || []) { - const criterionResult = extractAndReportCriterion({ - exp, - result, - content, - relFile, - packName: promptFile.pack, - promptId, - promptFilename: promptFile.filename, - meta, - outputFormat, - jsonFormatter, - verbose: !!verbose, - }); - - promptErrors += criterionResult.errors; - promptWarnings += criterionResult.warnings; - hadOperationalErrors = - hadOperationalErrors || criterionResult.hadOperationalErrors; - hadSeverityErrors = hadSeverityErrors || criterionResult.hadSeverityErrors; - criterionScores.push(criterionResult.scoreEntry); - - if (criterionResult.scoreComponent) { - scoreComponents.push(criterionResult.scoreComponent); - } - } - - if (outputFormat === OutputFormat.Json && scoreComponents.length > 0) { - (jsonFormatter as JsonFormatter | RdJsonFormatter).addEvaluationScore( - relFile, - { - id: promptId || promptFile.filename.replace(/\.md$/, ""), - scores: scoreComponents, - } + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a + // caller bypassed that boundary. Refuse it explicitly rather than + // misprojecting it as a check result. No adapter is added for judge metadata. + if (verbose) { + console.warn( + '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' ); } - - if (debugJson) { - const runId = randomUUID(); - const flat = result.criteria.flatMap((c) => - (c.violations || []).map((v, i) => ({ - criterion: c.name, - index: i, - violation: v, - decision: computeFilterDecision(v), - })) - ); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "judge", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: flat.map((x) => ({ - criterion: x.criterion, - index: x.index, - surface: x.decision.surface, - reasons: x.decision.reasons, - })), - surfaced_violations: flat.filter((x) => x.decision.surface).map((x) => ({ - criterion: x.criterion, - violation: x.violation, - })), - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } - return { - errors: promptErrors, - warnings: promptWarnings, - hadOperationalErrors, - hadSeverityErrors, - scoreEntries: criterionScores, + errors: 0, + warnings: 0, + hadOperationalErrors: true, + hadSeverityErrors: false, + scoreEntries: [], }; } diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index 73e56119..b4d2cf89 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -307,7 +307,11 @@ describe("CLI violation filtering", () => { expect(scoreLine).not.toContain("8.0/10"); }); - it("filters low-confidence judge violations from CLI counts by default", async () => { + it("rejects judge/rubric results instead of projecting them as check findings", async () => { + // Judge/rubric reviews are not a future-facing review type (Phase 3). The + // orchestrator refuses a JudgeResult rather than running the old criterion + // extraction/reporting path: no findings are projected and the run is + // flagged with an operational error. const targetFile = createTempFile("Alpha text\nBeta text\n"); const prompt = createPrompt({ id: "JudgePrompt", @@ -323,42 +327,17 @@ describe("CLI violation filtering", () => { makeJudgeViolation({ line: 2, quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", confidence: 0.2, }), ]) ); - const defaultRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(defaultRun.totalWarnings).toBe(1); + const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - process.env.CONFIDENCE_THRESHOLD = "0.0"; - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - description: "Issue 2", - analysis: "Issue 2", - suggestion: "Suggestion 2", - fix: "Fix 2", - confidence: 0.2, - }), - ]) - ); - - const zeroThresholdRun = await evaluateFiles( - [targetFile], - createBaseOptions([prompt]) - ); - expect(zeroThresholdRun.totalWarnings).toBe(2); + expect(run.totalErrors).toBe(0); + expect(run.totalWarnings).toBe(0); + expect(run.hadOperationalErrors).toBe(true); + expect(run.hadSeverityErrors).toBe(false); }); it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { From d4ea9a7764f682e1288a09303f7fa94d0c20c599 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:20 +0100 Subject: [PATCH 16/24] Remove orphaned orchestrator param types from CLI contract - Drop ProcessViolationsParams, ProcessCriterionParams, ProcessCriterionResult, and ValidationParams from src/cli/types.ts. They were orphaned when the orchestrator's inline violation/criterion processing moved to src/findings (Tasks 2-3) and the judge path was removed, and are referenced nowhere else. - Remove the now-unused imports PromptMeta, PromptCriterionSpec, ScoreComponent, and JudgeResult. Severity and PromptEvaluationResult are retained (still used by ReportIssueParams / ProcessPromptResultParams). --- src/cli/types.ts | 42 ++---------------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/src/cli/types.ts b/src/cli/types.ts index 7f95192a..a083f2e2 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -1,13 +1,12 @@ import type { PromptFile } from '../prompts/prompt-loader'; import type { LLMProvider } from '../providers/llm-provider'; import type { SearchProvider } from '../providers/search-provider'; -import type { PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; import type { FilePatternConfig } from '../boundaries/file-section-parser'; import type { EvaluationSummary } from '../output/reporter'; import { ValeJsonFormatter } from '../output/vale-json-formatter'; -import { JsonFormatter, type ScoreComponent } from '../output/json-formatter'; +import { JsonFormatter } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; -import type { PromptEvaluationResult, JudgeResult } from '../prompts/schema'; +import type { PromptEvaluationResult } from '../prompts/schema'; import { Severity } from '../evaluators/types'; import type { TokenUsageStats, PricingConfig } from '../providers/token-usage'; import type { Logger } from '../logging/logger'; @@ -95,43 +94,6 @@ export interface ReportIssueParams { match?: string; } -export interface ProcessViolationsParams extends EvaluationContext { - violations: Array<{ - line?: number; - quoted_text?: string; - context_before?: string; - context_after?: string; - message?: string; - analysis?: string; - suggestion?: string; - fix?: string; - }>; - severity: Severity; - ruleName: string; - scoreText: string; -} - -export interface ProcessCriterionParams extends EvaluationContext { - exp: PromptCriterionSpec; - result: JudgeResult; - packName: string; - promptId: string; - promptFilename: string; - meta: PromptMeta; -} - -export interface ProcessCriterionResult extends ErrorTrackingResult { - userScore: number; - maxScore: number; - scoreEntry: { id: string; scoreText: string; score?: number }; - scoreComponent?: ScoreComponent; -} - -export interface ValidationParams { - meta: PromptMeta; - result: JudgeResult; -} - export interface ProcessPromptResultParams extends EvaluationContext { promptFile: PromptFile; result: PromptEvaluationResult; From 15ec7a3e146df0820d39b9680e06b0369d5dd33e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:24:33 +0100 Subject: [PATCH 17/24] Document finding-processing behavior and clean stale judge docs - Add tests/findings/README.md mapping the behavior moved into src/findings (filtering, evidence verification, density scoring, rule-id/severity, formatter routing) and the intentionally removed judge/rubric and agent behavior, with agent tests marked out of scope for Phase 3 (Phase 3 Task 8). - Stop describing type: judge/subjective as a valid future-facing prompt type in CREATING_RULES.md, docs/frontmatter-fields.mdx, and docs/style-guide.mdx; note that judge/subjective are now rejected at load. - Drop the stale rubric-based scoring feature blurb from README.md. --- CREATING_RULES.md | 197 +++--------------------------------- README.md | 3 +- docs/frontmatter-fields.mdx | 80 ++------------- docs/style-guide.mdx | 113 ++------------------- tests/findings/README.md | 83 +++++++++++++++ 5 files changed, 117 insertions(+), 359 deletions(-) create mode 100644 tests/findings/README.md diff --git a/CREATING_RULES.md b/CREATING_RULES.md index ec90954f..7b800c9d 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -8,7 +8,6 @@ A comprehensive guide to creating powerful, reusable content evaluations using V - [Rule Anatomy](#rule-anatomy) - [Evaluation Modes](#evaluation-modes) - [Check Rules](#check-rules) -- [Judge Rules](#judge-rules) - [Target Specification](#target-specification) - [Configuration Reference](#configuration-reference) - [Best Practices](#best-practices) @@ -25,7 +24,7 @@ VectorLint rules are Markdown files with YAML frontmatter that define how your c - **Rule = Prompt file** (`.md` file organized in rule packs) - **Pack** = Subdirectory containing related rules (typically named after a company/style guide) - **Criteria** = Individual quality checks within a rule -- **Score** = LLM-assigned rating (1-4 scale for judge, density-based for check) +- **Score** = Density-based quality score derived from violation count - **Severity** = How failures are reported (`error` or `warning`) @@ -74,27 +73,20 @@ project/ ## Evaluation Modes -VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in two distinct modes, determined by the `type` field: +VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in one mode, determined by the `type` field: | Mode | `type` | Use Case | Scoring | Output | | ---------- | ------- | ------------------------------------- | --------------------------- | ----------------------- | | **Check** | `check` | Pass/fail checks, counting violations | 10 points - 1 per violation | List of specific issues | -| **Judge** | `judge` | Multi-dimensional quality scoring | 0-4 scale per criterion | Weighted average score | -### When to Use Each +The deprecated `type: judge` (alias `subjective`) is no longer supported; a rule using it is rejected at load. -**Use Check when:** +### When to Use Check - You need to find specific errors (e.g., "Find all grammar mistakes") - The check is binary (Pass/Fail) for each item - You want a list of specific violations to fix -**Use Judge when:** - -- You're measuring quality on a spectrum (e.g., "How engaging is this?") -- You have multiple dimensions (Clarity, Tone, Depth) -- You need weighted importance (some criteria matter more) - --- ## Check Rules @@ -141,71 +133,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes --- -## Judge Rules - -Judge rules use weighted criteria and a 1-4 rubric for sophisticated quality measurement. - -### Structure - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator - -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator... [Your detailed instructions] - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit - -### Good - -Clear benefit but less specific impact - -... -``` - -### The 1-4 Scoring Scale - -VectorLint uses a **1-4 scale** for all judge criteria, which is then normalized to a 1-10 scale: - -| LLM Rating | Meaning | Normalized Score | -| :--------- | :-------- | :--------------- | -| **4** | Excellent | **10.0** | -| **3** | Good | **7.0** | -| **2** | Fair | **4.0** | -| **1** | Poor | **1.0** | - -### Score Calculation - -1. **Normalization**: We map the 1-4 rating to a 1-10 score using the formula: `1 + ((Rating - 1) / 3) * 9`. -2. **Weighted Average**: The final score is the weighted average of all normalized criterion scores. - -**Example:** - -- Criterion: "Value Communication" (weight=12) -- Rating: 3 (Good) -> Normalized: 7.0 -- Weighted Points: 7.0 \* 12 = 84 points - ---- - ## Target Specification The `target` field allows you to: @@ -243,27 +170,17 @@ target: ### Frontmatter Fields -| Field | Type | Required | Description | -| ------------- | ------------- | -------- | ------------------------------------------------------------------ | -| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | -| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | -| `type` | string | No | Mode: `judge` or `check` (default: `check`) | -| `id` | string | **Yes** | Unique identifier (used in error reporting) | -| `name` | string | **Yes** | Human-readable name | +| Field | Type | Required | Description | +| ------------- | ------------- | -------- | -------------------------------------------------------------- | +| `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | +| `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | +| `type` | string | No | Mode: `check` (default: `check`). `judge`/`subjective` rejected at load | +| `id` | string | **Yes** | Unique identifier (used in error reporting) | +| `name` | string | **Yes** | Human-readable name | | `severity` | string | No | `error` or `warning` (default: `warning`) | | `evaluateAs` | string | No | `document` or `chunk` - whether to evaluate content as a whole or in chunks (default: `chunk`) | | `target` | object | No | Content matching specification | -| `criteria` | array | **Yes\*** | List of evaluation criteria (\*required for judge) | - -### Criterion Fields - -| Field | Type | Required | Description | -| -------- | ------ | -------- | ------------------------------------------ | -| `name` | string | **Yes** | Human-readable criterion name | -| `id` | string | **Yes** | Unique identifier (PascalCase recommended) | -| `weight` | number | No | Importance weight (default: 1) | -| `target` | object | No | Criterion-specific content matching | --- @@ -288,25 +205,10 @@ You are a headline evaluator for developer blog posts. Assess whether the headli 2. Uses natural, conversational language (avoid buzzwords) 3. Creates curiosity without being clickbait -For each criterion, provide a score (0-4) and specific examples from the text. -``` - -### 2. **Use Meaningful Weights (Subjective)** - -Scale weights to reflect real-world importance: - -```yaml -criteria: - # Technical accuracy is critical - - name: Technical Accuracy - weight: 40 - - # Readability is important - - name: Readability - weight: 30 +For each violation, quote the exact text and suggest a concrete fix. ``` -### 3. **Provide Context in Prompts** +### 2. **Provide Context in Prompts** Help the LLM understand your domain: @@ -338,79 +240,6 @@ Check this content for grammar issues, spelling errors, and punctuation mistakes Report any errors found with specific examples. ``` -### Example 2: Headline Evaluator (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: Headline -name: Headline Evaluator - -severity: error -target: - regex: '^#\s+(.+)$' - flags: "mu" - group: 1 - required: true - suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 10 - - name: Language Authenticity - id: LanguageAuthenticity - weight: 5 ---- - -You are a headline evaluator. Assess the H1 headline for: - -1. **Value Communication** (10 points): Does it clearly state what the reader gains? -2. **Language Authenticity** (5 points): Does it use natural, conversational language? - -## RUBRIC - -# Value Communication - -### Excellent - -Specific, immediately appealing benefit clearly stated - -... -``` - -### Example 3: AI Pattern Detector (Judge) - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: AIPatterns -name: AI Pattern Detector - -severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 ---- - -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Buzzwords**: leverage, synergy, elevate -2. **Formulaic transitions**: Moreover, Furthermore - ... -``` - ## Resources - [VectorLint README](./README.md) - Installation and basic usage diff --git a/README.md b/README.md index f6511e6a..662ac7f0 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,9 @@ If you can write a prompt for it, you can lint it with VectorLint. ## Quality Scores -VectorLint scores your content using error density and a rubric-based system, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. +VectorLint scores your content using error density, enabling you to measure quality across documentation. This gives your team a shared understanding of which content needs attention and helps track improvements over time. - **Density-Based Scoring:** For errors that can be counted, scores are calculated based on **error density (errors per 100 words)**, making quality assessment fair across documents of any length. -- **Rubric-Based Scoring:** For more nuanced quality standards, like flow and completeness, scores are graded on a 1-4 rubric system and then normalized to a **1-10 scale**. ## How VectorLint Reduces False Positives diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index 82e7d93c..d840d7b9 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -27,14 +27,12 @@ Your LLM prompt goes here. | Field | Default | Description | |---|---|---| -| `specVersion` | _(none)_ | Rule spec version. Set to `1.0.0` for rules using the full judge schema. | +| `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | | `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | -| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` scores content against weighted criteria on a 1–4 rubric. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations and scores them by error density. `judge` (and its deprecated `subjective` alias) is no longer supported — a rule with `type: judge` or `type: subjective` is rejected at load. | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | | `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | | `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | -| `criteria` | _(none)_ | Required when `type` is `judge`. Defines the scoring dimensions and their weights. See [Criteria fields](#criteria-fields) below. | - ## Target fields The `target` field narrows evaluation to a specific part of the document. It is an object with the following sub-fields: @@ -58,40 +56,16 @@ target: suggestion: Add an H1 headline for the article. ``` -## Criteria fields - -The `criteria` field is an array of criterion objects used with `type: judge`. Each criterion defines one scoring dimension. - -| Field | Type | Required | Description | -|---|---|---|---| -| `name` | string | Yes | Human-readable criterion name shown in CLI output. | -| `id` | string | Yes | Unique identifier for this criterion. PascalCase recommended (e.g., `TechnicalAccuracy`). Referenced in the Markdown body rubric as `# CriterionName `. | -| `weight` | number | No | Relative importance of this criterion in the weighted average. Higher values increase the criterion's influence on the final score. Defaults to `1`. | -| `target` | object | No | Criterion-specific content matching. Uses the same sub-fields as the top-level `target`. Overrides the rule-level target for this criterion only. | - -**Example — judge rule with two weighted criteria:** - -```yaml -criteria: - - name: Technical Accuracy - id: TechnicalAccuracy - weight: 40 - - name: Readability - id: Readability - weight: 30 -``` - ## Complete example -The following rule uses all commonly-used fields. It targets the H1 headline, judges it against two criteria, and requires the headline to exist before evaluation runs. +The following rule uses the most commonly-used fields. It targets the H1 headline and requires the headline to exist before evaluation runs. ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator +type: check +id: HeadlineQuality +name: Headline Quality severity: error evaluateAs: document target: @@ -100,44 +74,10 @@ target: group: 1 required: true suggestion: Add an H1 headline for the article. -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 --- -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. +You are a headline reviewer for developer blog posts. Evaluate the H1 headline +and report specific, fixable violations — for example: a vague benefit, a +missing subject, or passive voice. For each violation, quote the exact text and +suggest a concrete rewrite. ``` diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index 1fcaccb1..3dfe0c20 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -79,21 +79,19 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| | `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | -| `type` | `check` | Evaluation mode: `check` or `judge`. See below. | +| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` (and its `subjective` alias) is no longer supported and is rejected at load. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | | `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | -| `criteria` | _(none)_ | Required for `judge` rules. Defines the scoring dimensions. | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | ## Evaluation Modes -VectorLint supports two evaluation modes, chosen with the `type` field. +VectorLint uses a single evaluation mode, set with the `type` field. | Mode | `type` value | Best for | |------|-------------|----------| | Check | `check` | Finding specific, countable violations (grammar errors, banned terms) | -| Judge | `judge` | Measuring quality on a spectrum (clarity, tone, completeness) | ### Check rules @@ -122,69 +120,6 @@ Report any errors found with specific examples. | `standard` (4–7) | ~10 points | | `strict` (8–10) | ~20 points | -### Judge rules - -The LLM scores content against multiple weighted criteria using a 1–4 rubric. VectorLint normalizes each score to a 1–10 scale and computes a weighted average. - -| LLM rating | Meaning | Normalized | -|------------|---------|------------| -| 4 | Excellent | 10.0 | -| 3 | Good | 7.0 | -| 2 | Fair | 4.0 | -| 1 | Poor | 1.0 | - -Define criteria in the frontmatter and expand each into a rubric section in the Markdown body: - -```markdown ---- -specVersion: 1.0.0 -evaluator: base -type: judge -id: HeadlineEvaluator -name: Headline Evaluator -severity: error -criteria: - - name: Value Communication - id: ValueCommunication - weight: 12 - - name: Curiosity Gap - id: CuriosityGap - weight: 2 ---- - -You are a headline evaluator for developer blog posts. - -## RUBRIC - -# Value Communication - -### Excellent -Specific, immediately appealing benefit clearly stated. - -### Good -Clear benefit but less specific impact. - -### Fair -Vague benefit implied but not stated. - -### Poor -No apparent benefit to the reader. - -# Curiosity Gap - -### Excellent -Creates genuine intrigue without being misleading. - -### Good -Mildly interesting, reader may continue. - -### Fair -Neutral — no curiosity created. - -### Poor -Actively off-putting or confusing. -``` - ## Targeting Specific Content The `target` field lets you evaluate a specific portion of a document — for example, only the H1 headline — rather than the full content. @@ -227,51 +162,23 @@ RunRules=Acme ## Examples -### AI Pattern Detector (Judge) +### AI Pattern Detector (Check) ```markdown --- -specVersion: 1.0.0 evaluator: base -type: judge +type: check id: AIPatterns name: AI Pattern Detector severity: warning -criteria: - - name: Language Authenticity - id: LanguageAuthenticity - weight: 40 - - name: Structural Naturalness - id: StructuralNaturalness - weight: 30 --- -Detect AI-generated writing patterns in this content. - -## INSTRUCTION - -Scan for common AI patterns: - -1. **Overused buzzwords**: leverage, synergy, elevate, unlock, empower -2. **Formulaic transitions**: Moreover, Furthermore, In conclusion -3. **Hollow openings**: "In today's fast-paced world..." -4. **Excessive hedging**: "It's worth noting that...", "It's important to mention..." - -## RUBRIC - -# Language Authenticity - -### Excellent -Natural, human voice throughout. No AI patterns detected. - -### Good -Mostly natural. One or two minor patterns present. - -### Fair -Several AI patterns reduce authenticity. - -### Poor -Pervasive AI patterns throughout. +Detect AI-generated writing patterns in this content. Report each occurrence as a +violation: overused buzzwords (leverage, synergy, elevate, unlock, empower), +formulaic transitions (Moreover, Furthermore, In conclusion), hollow openings +("In today's fast-paced world..."), and excessive hedging ("It's worth noting +that..."). For each violation, quote the exact text and suggest a concrete, +human-sounding rewrite. ``` ## Next Steps diff --git a/tests/findings/README.md b/tests/findings/README.md new file mode 100644 index 00000000..f48cc664 --- /dev/null +++ b/tests/findings/README.md @@ -0,0 +1,83 @@ +# Finding Processing — Test Coverage & Behavior Mapping + +This directory owns the regression coverage for the **shared objective +violation-check finding-processing pipeline** (`src/findings/`). It documents +which behaviors moved out of `src/cli/orchestrator.ts` (and adjacent output +tests) into `src/findings/`, which behaviors were **intentionally removed**, and +which behaviors are **out of scope** for this phase. + +The pipeline (`processFindings`) composes a single path: + +```text +candidate findings ──▶ verify evidence ──▶ filter ──▶ dedupe ──▶ score ──▶ severity ──▶ ReviewResult +``` + +It is independent of model call (`single` / `agent` / `auto`) and carries no +`judge`, `evaluator`, rubric, or autonomous-agent compatibility path. + +--- + +## Behavior-preservation mapping + +| Behavior (origin) | Preserved by | Notes | +|---|---|---| +| Violation filtering / surfacing (`orchestrator.ts` `computeFilterDecision`) | `processor.test.ts` — *drops candidates that fail computeFilterDecision without a diagnostic*; *filtered candidates do not contribute to the score count* | Filter logic itself stays in `src/debug/violation-filter.ts`; the processor is a caller. `orchestrator-filtering.test.ts` still guards the end-to-end CLI surfacing. | +| Fuzzy finding-evidence location (`locateQuotedText`) | `finding-evidence-verifier.test.ts` — exact-quote location, warn diagnostic on miss, **no model-line fallback**, anchors without a line hint, diagnostic names the quoted text | One verifier (audit Finding #6). The agent path's model-line fallback is **not** preserved (see Removed). | +| Verified-finding counting & density scoring (`calculateCheckScore`) | `scorer.test.ts` — same numeric result as `calculateCheckScore`; score driven by verified count, not raw candidate count; resolves error severity; forwards strictness/severity | `scorer.ts` is a thin wrapper; no reimplemented math. | +| Severity + rule-id attribution | `severity.test.ts` — `resolveSeverity`, `buildRuleId` (`Pack.Rule` / `Pack.Rule.Criterion`), `resolveCriterionId`; `processor.test.ts` — stamps severity, attributes to pack rule / criterion | One severity path; one rule-id builder (audit Finding #4). | +| Golden check output (byte-for-byte) | `processor.test.ts` — *produces byte-for-byte findings/score matching the standard orchestrator path* | Proves no regression before the orchestrator was rewired (Task 2). | +| Unanchored-evidence diagnostics & exit behavior | `processor.test.ts` — *turns unanchored evidence into a warn diagnostic and emits no finding*; *counts only verified findings toward the score*; *does not set hadOperationalErrors for warn-level evidence diagnostics* | Intentional Phase 3 fix: unanchored quotes no longer flag the run as operationally failed. | +| Deduplication | `processor.test.ts` — *deduplicates verified findings by quoted_text and line* | | +| Contract / input boundary | `types.test.ts` — `FINDING_PROCESSING_INPUT_SCHEMA`, `RAW_VIOLATION_SCHEMA`, `PROMPT_META_FOR_FINDINGS_SCHEMA` parse supported input and reject legacy `evaluator`/`judge`/`rubric weight`/`agent`/`modelCall`/`mode`; `module-surface.test.ts` — public surface + diagnostic code constant | Strict Zod at the boundary; no `any`. | +| Formatter routing (line / JSON / RDJSON / Vale) | `orchestrator-check-processor.test.ts` — locatable findings + score + counts unchanged; verified findings through the JSON sink; unanchored quotes omitted from Vale JSON. `orchestrator-filtering.test.ts` — no dummy JSON/Vale issues when nothing is surfaced | End-to-end coverage of the rewired check path feeding existing formatters from `ReviewResult`. | + +--- + +## Intentionally removed behavior (no replacement fixture) + +These were legacy paths that the Phase 3 contract no longer supports. They are +covered by **rejection tests**, not behavior-parity tests. + +| Removed behavior | Rejection coverage | +|---|---| +| Subjective `judge` / rubric criterion evaluation | `prompt-loader-validation.test.ts` — *Judge boundary rejection (Phase 3)* rejects `type: judge` and the `subjective` alias at the loader boundary; `orchestrator-filtering.test.ts` — *rejects judge/rubric results instead of projecting them as check findings* | +| Legacy `evaluator` / `judge` / rubric criteria in the review contract | `review/types.test.ts` — *rejects legacy evaluator and judge criteria fields*; `findings/types.test.ts` — input contract rejects judge/evaluator/rubric shapes | +| Agent finding-evidence model-line fallback | `finding-evidence-verifier.test.ts` — *never returns a finding whose line came from the model when the quote did not anchor* (the removed behavior is asserted as absent) | + +Judge criterion output behavior (1–4 rubric, weighted average) has **no** +replacement fixture in this phase. Objective rule coverage belongs to the +objective-check test inventory, not a ported rubric. + +--- + +## Out of scope for Phase 3 (Phase 4 territory) + +These are **not** behavior-preservation targets for this phase and are +intentionally left untouched: + +- `tests/agent/*`, the orchestrator test covering legacy agent output, and + `tests/providers/*agent-loop*.test.ts` — the old autonomous agent surface. + Phase 4 deletes or replaces these with agent-model-call executor tests. +- `BaseEvaluator`'s internal judge mode (`runJudgeEvaluation`), + `buildJudgeLLMSchema` (`src/prompts/schema.ts`), the judge scoring helpers in + `src/scoring/`, `tests/scoring-types.test.ts`, and the judge cases in + `tests/prompt-schema.test.ts` — evaluator internals that are now **unreachable + from the CLI** (judge prompts fail to load) but are scheduled for deletion in + Phase 4 (executors / provider interface / agent deletion). + +Phase 3 added **no** compatibility code for the legacy agent surface — no +agent tool-loop wiring, no review-instruction handler, and no agent result +projection — to `src/findings/`. + +--- + +## Files + +| File | Covers | +|---|---| +| `finding-evidence-verifier.test.ts` | Single evidence verifier; no model-line fallback. | +| `severity.test.ts` | `resolveSeverity`, `buildRuleId`, `resolveCriterionId`. | +| `scorer.test.ts` | `scoreCheck` numerics vs `calculateCheckScore`; verified-count scoring. | +| `types.test.ts` | Zod input contract: parses supported input, rejects legacy shapes. | +| `processor.test.ts` | `processFindings` golden output, diagnostics, counting fix, dedupe, rule-id attribution, `ReviewResult` validation. | +| `module-surface.test.ts` | Public barrel surface + stable diagnostic code constant. | From 4fa07011105f3f2b84df3ae22d41fd774e27749e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 08:50:55 +0100 Subject: [PATCH 18/24] Sanitize prompt criteria at the findings boundary - Map meta.criteria to { id, name } before processFindings so legacy PromptCriterionSpec rubric fields (weight, target) cannot cross the standard check orchestration -> findings contract - Add tests/orchestrator-finding-criteria.test.ts mocking processFindings to assert weight/target are stripped on the production path - prompt-loader criteria support is unchanged; criterion id/name still drive output rule-id attribution --- src/cli/orchestrator.ts | 6 +- tests/orchestrator-finding-criteria.test.ts | 126 ++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 tests/orchestrator-finding-criteria.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 4d09e917..1e61a47d 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -206,7 +206,11 @@ function routePromptResult( promptMeta: { ...(meta.severity !== undefined ? { severity: meta.severity } : {}), ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - ...(meta.criteria ? { criteria: meta.criteria } : {}), + // Sanitize to the findings contract ({ id, name }): meta.criteria is + // PromptCriterionSpec[], which can carry legacy rubric weight/target. + ...(meta.criteria + ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } + : {}), }, targetContent: content, }); diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts new file mode 100644 index 00000000..dc0b1ad8 --- /dev/null +++ b/tests/orchestrator-finding-criteria.test.ts @@ -0,0 +1,126 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import path from "path"; +import { evaluateFiles } from "../src/cli/orchestrator"; +import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; +import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { PromptFile } from "../src/prompts/prompt-loader"; +import type { RawCheckResult } from "../src/prompts/schema"; +import type { ReviewResult } from "../src/review/types"; + +const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ + EVALUATE_MOCK: vi.fn(), + PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), +})); + +// createEvaluator returns a controllable evaluator so evaluateFiles reaches +// routePromptResult -> processFindings without a real model call. +vi.mock("../src/evaluators/index", () => ({ + createEvaluator: vi.fn(() => ({ + evaluate: EVALUATE_MOCK, + })), +})); + +// Replace only processFindings (the findings boundary); keep every other +// findings export real so unrelated imports stay intact. +vi.mock("../src/findings", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; +}); + +function createPrompt(meta: PromptFile["meta"]): PromptFile { + return { + id: meta.id, + filename: `${meta.id}.md`, + fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), + meta, + body: "Prompt body", + pack: "TestPack", + }; +} + +function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { + return { + prompts, + rulesPath: undefined, + provider: {} as never, + concurrency: 1, + verbose: false, + debugJson: false, + scanPaths: [], + outputFormat: OutputFormat.Line, + }; +} + +function createTempFile(content: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); + const filePath = path.join(dir, "input.md"); + writeFileSync(filePath, content); + return filePath; +} + +function makeCheckResult(): RawCheckResult { + return { + type: EvaluationType.CHECK, + violations: [], + word_count: 100, + }; +} + +describe("standard check orchestration sanitizes criteria at the findings boundary", () => { + beforeEach(() => { + EVALUATE_MOCK.mockReset(); + PROCESS_FINDINGS_MOCK.mockReset(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("drops rubric weight/target from criteria before reaching processFindings", async () => { + const targetFile = createTempFile("Alpha text\n"); + // PromptCriterionSpec admits legacy rubric fields (weight, target). The + // orchestrator must strip them so only { id, name } crosses the boundary. + const prompt = createPrompt({ + id: "SanitizePrompt", + name: "Sanitize Prompt", + type: "check", + severity: Severity.WARNING, + criteria: [ + { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, + ], + }); + + EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); + PROCESS_FINDINGS_MOCK.mockReturnValue({ + findings: [], + scores: [ + { + ruleId: "TestPack.SanitizePrompt", + score: 10, + scoreText: "10.0/10", + severity: "warning", + }, + ], + diagnostics: [], + }); + + await evaluateFiles([targetFile], createBaseOptions([prompt])); + + expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); + const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { + promptMeta: { criteria?: Array> }; + }; + + // toEqual fails if any extra (weight/target) key survives the mapping. + expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); + for (const criterion of input.promptMeta.criteria ?? []) { + expect(criterion).not.toHaveProperty("weight"); + expect(criterion).not.toHaveProperty("target"); + } + }); +}); From 99e33d6acf6072c27ee437af2c3be509013f298a Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 14:53:46 +0100 Subject: [PATCH 19/24] Reframe agent-mode messaging as unreleased internal path - README: replace the "Agent Mode (under review)" deprecation section with "Internal Agent-Mode Implementation" framing (unreleased/internal path, no public compatibility contract) - orchestrator: reword the quarantine comment and the --mode agent runtime warning to unreleased/internal framing; fallback behavior unchanged - CLI --help: reword the --mode option text away from "deprecated" - findings module/verifier + tests: replace old/legacy/surface/compatibility wording with unreleased/internal framing; technical assertions unchanged - orchestrator-agent-output tests: update warning assertions to the new text (forced by the sanctioned warning reword) and align labels/comments No public deprecation, migration, or compatibility obligation is implied. --- README.md | 11 +++-------- src/cli/commands.ts | 2 +- src/cli/orchestrator.ts | 7 ++++--- src/findings/finding-evidence-verifier.ts | 4 ++-- src/findings/index.ts | 4 ++-- tests/findings/README.md | 4 ++-- tests/findings/finding-evidence-verifier.test.ts | 5 +++-- tests/orchestrator-agent-output.test.ts | 12 ++++++------ 8 files changed, 23 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 662ac7f0..d0b7d0b7 100644 --- a/README.md +++ b/README.md @@ -144,16 +144,11 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode (under review) +## Internal Agent-Mode Implementation -The `--mode agent` flag is under active rework. It currently enables an -autonomous cross-file review mode that is being **removed** in favor of a -bounded harness model. See -[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) -for the decision and the refactor plan. +The `--mode agent` flag is an unreleased internal implementation path retained only while the harness refactor lands. It now falls back to standard evaluation and reports an internal-status warning so the old autonomous workspace implementation is not used. -Until the refactor lands, `--mode agent` prints a deprecation warning and -falls back to standard mode. Do not build integrations against it. +This path has no public compatibility contract; do not build integrations against it. ## Contributing diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 50dbffc8..5af380c6 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -82,7 +82,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) + .option('--mode ', 'Execution mode: standard (default). "agent" is an unreleased internal path and falls back to standard.', DEFAULT_REVIEW_MODE) .option('-p, --print', 'Suppress interactive progress output in agent mode') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 1e61a47d..d1526302 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -657,8 +657,9 @@ async function buildAgentRuleScores( return results; } -// Retained in quarantine: unreachable from the CLI after --mode agent was -// deprecated (now falls back to standard evaluation). Removed in Phase 4. +// Retained in quarantine: the unreleased internal agent-mode implementation is +// unreachable from the CLI because --mode agent falls back to standard evaluation. +// Removed in Phase 4. // eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine async function evaluateFilesInAgentMode( targets: string[], @@ -815,7 +816,7 @@ export async function evaluateFiles( if (mode === AGENT_REVIEW_MODE) { options.logger?.warn( - '--mode agent is deprecated and now falls back to standard mode. ' + + '--mode agent is an unreleased internal path and now falls back to standard mode. ' + 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', ); // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. diff --git a/src/findings/finding-evidence-verifier.ts b/src/findings/finding-evidence-verifier.ts index 7a6ffb6a..e2d6df9e 100644 --- a/src/findings/finding-evidence-verifier.ts +++ b/src/findings/finding-evidence-verifier.ts @@ -44,8 +44,8 @@ export interface FindingEvidenceVerification { * matching. On a successful anchor it returns a verified finding with the * located line/column/match. When the quoted text cannot be anchored it * returns a `finding-evidence-not-locatable` warn diagnostic and NO finding — - * it never falls back to a model-provided line number the way the old agent - * path did. + * it never falls back to a model-provided line number the way the unreleased + * internal agent implementation did. */ export function verifyFindingEvidence( content: string, diff --git a/src/findings/index.ts b/src/findings/index.ts index 095b878c..27a585cf 100644 --- a/src/findings/index.ts +++ b/src/findings/index.ts @@ -6,8 +6,8 @@ * count/density scoring, severity resolution, and diagnostics. * * This module is independent of model call (`single` | `agent`) and contains - * no legacy `judge`, `evaluator`, rubric, or autonomous-loop output - * compatibility code. See + * no `judge`, `evaluator`, rubric, or autonomous-loop output + * adapter code. See * docs/plans/2026-07-10-phase-3-share-result-projection.md. */ export { diff --git a/tests/findings/README.md b/tests/findings/README.md index f48cc664..c856a2b4 100644 --- a/tests/findings/README.md +++ b/tests/findings/README.md @@ -56,7 +56,7 @@ These are **not** behavior-preservation targets for this phase and are intentionally left untouched: - `tests/agent/*`, the orchestrator test covering legacy agent output, and - `tests/providers/*agent-loop*.test.ts` — the old autonomous agent surface. + `tests/providers/*agent-loop*.test.ts` — the unreleased internal autonomous agent implementation. Phase 4 deletes or replaces these with agent-model-call executor tests. - `BaseEvaluator`'s internal judge mode (`runJudgeEvaluation`), `buildJudgeLLMSchema` (`src/prompts/schema.ts`), the judge scoring helpers in @@ -65,7 +65,7 @@ intentionally left untouched: from the CLI** (judge prompts fail to load) but are scheduled for deletion in Phase 4 (executors / provider interface / agent deletion). -Phase 3 added **no** compatibility code for the legacy agent surface — no +Phase 3 added **no** compatibility code for the unreleased/internal agent implementation — no agent tool-loop wiring, no review-instruction handler, and no agent result projection — to `src/findings/`. diff --git a/tests/findings/finding-evidence-verifier.test.ts b/tests/findings/finding-evidence-verifier.test.ts index 64d38209..28422d16 100644 --- a/tests/findings/finding-evidence-verifier.test.ts +++ b/tests/findings/finding-evidence-verifier.test.ts @@ -34,8 +34,9 @@ describe('verifyFindingEvidence', () => { }); it('never returns a finding whose line came from the model when the quote did not anchor', () => { - // The old agent path fell back to the model-provided line (here 1) and - // surfaced the raw quoted_text as a verified match. Assert we do not. + // The unreleased internal agent implementation fell back to the + // model-provided line (here 1) and surfaced the raw quoted_text as a + // verified match. Assert we do not. // (Use a lexically distant quote that locateQuotedText cannot anchor.) const result = verifyFindingEvidence(CONTENT, { quoted_text: 'definitely not present xyzzy', diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index 0b61ea63..1dab5b95 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -54,11 +54,11 @@ function makeStandardProvider(): StandardProviderSpies { return { provider, runPromptStructured, runAgentToolLoop }; } -// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no +// `--mode agent` is an unreleased internal path. These tests prove the CLI/evaluateFiles path no // longer reaches the autonomous agent executor: it warns through the injected // logger and falls back to standard evaluation. The retained agent executor // code is covered directly by tests/agent/* and removed in Phase 4. -describe('agent mode deprecation', () => { +describe('agent mode fallback', () => { const tempRepos: string[] = []; function createTempRepo(): string { @@ -80,7 +80,7 @@ describe('agent mode deprecation', () => { } }); - it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { + it('emits an internal-status warning through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); @@ -102,7 +102,7 @@ describe('agent mode deprecation', () => { }); expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('deprecated'), + expect.stringContaining('unreleased internal path'), ); expect(logger.warn).toHaveBeenCalledWith( expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), @@ -134,7 +134,7 @@ describe('agent mode deprecation', () => { logger, }); - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('unreleased internal path')); expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(runPromptStructured).toHaveBeenCalled(); }); @@ -158,7 +158,7 @@ describe('agent mode deprecation', () => { scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], }); - // No deprecation warning and no executor invocation in standard mode. + // No fallback warning and no executor invocation in standard mode. expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(result.totalFiles).toBe(1); }); From 716a77c120cae6f3db8d7df309dda17080d8a9a0 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 15:15:04 +0100 Subject: [PATCH 20/24] refactor(vectorlint): remove committed planning references - Keep agent mode warnings product-facing and internal-only\n- Remove obsolete audit and plan assertions from the correction scope --- src/cli/orchestrator.ts | 3 +-- src/findings/index.ts | 3 +-- tests/orchestrator-agent-output.test.ts | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index d1526302..aaedc3cb 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -816,8 +816,7 @@ export async function evaluateFiles( if (mode === AGENT_REVIEW_MODE) { options.logger?.warn( - '--mode agent is an unreleased internal path and now falls back to standard mode. ' + - 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', + '--mode agent is an unreleased internal path and now falls back to standard mode.', ); // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } diff --git a/src/findings/index.ts b/src/findings/index.ts index 27a585cf..c3a9a666 100644 --- a/src/findings/index.ts +++ b/src/findings/index.ts @@ -7,8 +7,7 @@ * * This module is independent of model call (`single` | `agent`) and contains * no `judge`, `evaluator`, rubric, or autonomous-loop output - * adapter code. See - * docs/plans/2026-07-10-phase-3-share-result-projection.md. + * adapter code. */ export { FINDING_EVIDENCE_NOT_LOCATABLE, diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index 1dab5b95..b6c8e271 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -104,9 +104,6 @@ describe('agent mode fallback', () => { expect(logger.warn).toHaveBeenCalledWith( expect.stringContaining('unreleased internal path'), ); - expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), - ); // Standard evaluation ran; the autonomous executor did not. expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(runPromptStructured).toHaveBeenCalled(); From 8f0680b1b9d26e2319a79f790bbaa7519a01452d Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 15:16:11 +0100 Subject: [PATCH 21/24] refactor(cli): reframe agent mode as internal - Describe agent mode as an unreleased internal implementation path\n- Keep the standard-mode fallback notice focused on bounded harness rework\n- Remove the audit artifact and align orchestrator coverage with the notice --- README.md | 13 +- ...0-vectorlint-harness-architecture-audit.md | 380 ------------------ src/cli/commands.ts | 2 +- src/cli/orchestrator.ts | 8 +- tests/orchestrator-agent-output.test.ts | 14 +- 5 files changed, 17 insertions(+), 400 deletions(-) delete mode 100644 docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md diff --git a/README.md b/README.md index f6511e6a..4e1fba62 100644 --- a/README.md +++ b/README.md @@ -145,16 +145,13 @@ Notes: - Prompts and outputs are recorded when Langfuse observability is enabled. - Do not send secrets, credentials, or PII unless your policy explicitly allows observability tooling to access that data. -## Agent Mode (under review) +## Agent Mode (internal rework) -The `--mode agent` flag is under active rework. It currently enables an -autonomous cross-file review mode that is being **removed** in favor of a -bounded harness model. See -[`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`](docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md) -for the decision and the refactor plan. +The `--mode agent` flag was an unreleased internal implementation path. It is +being replaced by a bounded harness model in this refactor. -Until the refactor lands, `--mode agent` prints a deprecation warning and -falls back to standard mode. Do not build integrations against it. +While the refactor lands, `--mode agent` prints an internal-rework notice and +falls back to standard mode. Do not treat it as a public integration surface. ## Contributing diff --git a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md b/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md deleted file mode 100644 index 51161ce0..00000000 --- a/docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md +++ /dev/null @@ -1,380 +0,0 @@ -# VectorLint Harness Architecture Audit - -Date: 2026-07-10 - -Status: Objective system audit plus product-direction audit. Request changes before shipping the current autonomous workspace-agent surface as a public contract. - -## Product Decision - -VectorLint should not keep the current autonomous workspace-agent mode. - -The retained agent-like capability should be a constrained reader executor: a model or headless agent can read the target content in sections when that is useful for context management, but it should not search the workspace, inspect arbitrary files, perform top-level workspace checks, or rewrite the rule being evaluated. - -The intended execution model is: - -- `direct`: send target content and rule in one structured review call. -- `reader`: give the executor a read-section capability over the target content only. -- `auto`: choose `direct` for normal-sized inputs and `reader` for large inputs or explicit context-management needs. - -This is an execution strategy decision, not a return to VectorLint-as-agent. - -## Product Direction Under Review - -VectorLint should become a programmable, bounded, on-page content review harness that another caller can invoke. - -The caller can be Codex, Claude Code, another coding agent, CI, a local CLI wrapper, or a future service. That caller owns exploration, cross-page reasoning, research, and context gathering. VectorLint owns the constrained review of target content against source-backed rules and returns structured findings, scores, diagnostics, and usage metadata. - -In practical terms: - -- VectorLint reviews the target page or explicitly supplied content. -- External context is caller-supplied, not discovered by VectorLint. -- Rules define review behavior, granularity, severity, and scoring constraints. -- Executors can be API models, headless local agents, or constrained reader executors, but they receive the same review request contract. -- VectorLint should not expose a model-controlled workspace tool loop as its core product surface. - -## Objective System Audit Summary - -Separate from product direction, the current codebase has several system-level liabilities: - -- Build health is weak: `npx tsc --noEmit`, `npm run lint`, and parts of `npm run test:run` fail in the current workspace. -- Runtime contracts are duplicated across hand-written TypeScript, JSON-schema objects, Zod schemas, and formatter-specific shapes. -- Standard mode and agent mode project the same underlying findings differently. -- CLI orchestration owns too many responsibilities: config, file matching, evaluation, scoring, output formatting, debug artifacts, and agent routing. -- Evidence verification and counting are inconsistent enough to affect exit behavior. -- Observability and debug paths can record full prompts, content, and outputs. -- Documentation and implementation disagree on the current agent-mode topology and tool contracts. - -## Architecture Verdict - -The current codebase contains two execution models at once: - -1. The older evaluator path: one structured model call per rule or chunk. -2. The newer autonomous workspace-agent path: VectorLint gives a model tools to read/search the workspace and call lint as a nested tool. - -The second path should not continue as-is. The useful piece is not "agent mode" broadly; it is target-aware reading for context management. The current implementation also duplicates result projection, scoring, evidence validation, output formatting, and configuration behavior from the first path. That is why the architecture feels harder to maintain: the code is not just messy; it is serving two different ownership models. - -The highest-leverage change is to define a neutral review-domain contract and make every executor implement that contract. - -## Proposed Core Contract - -The contract should be centered on review, not providers or agents. - -```ts -interface ReviewRequest { - target: ReviewTarget; - rules: ReviewRule[]; - context?: ReviewContext[]; - budget: ReviewBudget; - outputPolicy: ReviewOutputPolicy; -} - -interface ReviewExecutor { - run(request: ReviewRequest): Promise; -} - -interface ReviewResult { - findings: ReviewFinding[]; - scores: ReviewScore[]; - diagnostics: ReviewDiagnostic[]; - usage?: ReviewUsage; -} -``` - -API models, reasoning models, and headless CLI agents become adapter implementations behind this contract. The CLI becomes orchestration around request creation, result formatting, and exit behavior. - -## Major Findings - -### 1. The Current Agent Surface Is Broader Than The Decided Execution Model - -Evidence: - -- `README.md:148` presents `--mode agent` as autonomous cross-file review mode. -- `src/agent/prompt-builder.ts:43` instructs the model to perform workspace-level checks. -- `src/agent/executor.ts:670` exposes `read_file`. -- `src/agent/executor.ts:715` exposes `search_content`. - -Impact: - -VectorLint still behaves like it owns exploration. That puts it in competition with external coding agents instead of becoming a harness those agents can use. It also overshoots the decided reader-executor model, which only needs target-scoped reading for context management. - -Recommendation: - -Remove the current autonomous workspace-agent surface. Preserve only a constrained reader executor if needed: - -- It may read sections of the target content. -- It may not search the workspace. -- It may not read arbitrary files. -- It may not perform top-level workspace findings. -- It may not override source-backed rules. - -### 2. The Provider Interface Mixes Structured Review With Autonomous Tool Loops - -Evidence: - -- `src/providers/llm-provider.ts:28` requires both `runPromptStructured` and `runAgentToolLoop`. -- `src/providers/vercel-ai-provider.ts:136` implements the agent loop on the same provider. -- `src/agent/executor.ts:796` depends on the provider-level agent loop. - -Impact: - -Every future executor has to pretend to be both a structured model provider and an autonomous agent runner. That is the wrong abstraction for headless CLI support and for a constrained reader executor. - -Recommendation: - -Split the contracts: - -- `StructuredModelClient` for API structured output. -- `HeadlessReviewExecutor` for local CLI agents. -- `ReaderReviewExecutor` for target-scoped read-section execution. -- `ReviewExecutor` as the stable domain-level interface. -- `TelemetrySink` or observability decoration as an optional cross-cutting concern. - -### 3. Runtime Type Contracts Are Not Strong Enough For A Harness - -Evidence: - -- `src/providers/vercel-ai-provider.ts:119` returns `output as T`. -- `src/providers/llm-provider.ts:9` defines tool input and output as `unknown`. -- `src/agent/tools-registry.ts:21` exposes tools without typed output contracts. -- `npx tsc --noEmit` currently fails in core files. - -Impact: - -The code presents types as guarantees, but several are only assertions. A programmable harness needs reliable runtime validation because external callers and headless adapters will depend on these contracts. - -Recommendation: - -Tie structured execution to concrete Zod schemas or canonical parsers. Introduce typed tool contracts only if tools remain. Add a `typecheck` script and wire it into CI and build. - -### 4. Standard Mode And Agent Mode Project Results Differently - -Evidence: - -- `src/cli/orchestrator.ts:626` derives standard check severity from scoring. -- `src/agent/executor.ts:431` stamps agent findings with prompt severity. -- `src/agent/executor.ts:598` flattens judge criteria before recording findings. -- `src/cli/orchestrator.ts:1216` builds agent scores only for line output. -- `src/cli/orchestrator.ts:1237` emits structured output without agent scores or diagnostics. - -Impact: - -The same underlying review can produce different severity, rule identity, score data, JSON shape, and exit behavior depending on mode. That makes VectorLint hard to trust as a machine-facing gate. - -Recommendation: - -Extract one shared result projection pipeline: - -```text -PromptEvaluationResult - -> verified findings - -> rule and criterion identity - -> severity and score - -> diagnostics - -> output formatter -``` - -Every executor should return into this pipeline. - -### 5. The On-Page Boundary Is Not Enforced - -Evidence: - -- `src/agent/executor.ts:585` lets `lint` resolve any file inside `workspaceRoot`. -- `src/agent/executor.ts:670` lets `read_file` read any workspace file. -- `src/agent/executor.ts:715` lets `search_content` scan workspace content. -- `src/agent/executor.ts:134` allows model-supplied `reviewInstruction`. -- `src/agent/executor.ts:583` replaces the source-backed prompt body with the model-supplied override. - -Impact: - -Prompt-injected page content can indirectly steer the model to read non-target files or rewrite the rule being evaluated. That breaks deterministic review semantics and creates privacy and security risk. - -Recommendation: - -For the new harness, enforce a target/context allowlist: - -- Target content is always in scope. -- Caller-supplied context is in scope. -- Workspace reads are out of scope unless the caller explicitly includes those files as context. -- Reader execution can page through target content only. -- Rule bodies are source-backed and caller-authored, not model-authored. - -### 6. Evidence Handling Can Misreport Findings - -Evidence: - -- Standard mode skips unverifiable quotes but can still count original surfaced violation totals in some paths. -- `src/agent/executor.ts:415` attempts to locate evidence. -- `src/agent/executor.ts:426` falls back to the model-provided line when location fails. - -Impact: - -Standard mode can fail a run without printing the issue that caused the failure. Agent mode can surface hallucinated evidence as if it were verified. - -Recommendation: - -Use one evidence verifier. Count only emitted, verified findings. If evidence cannot be located, return a diagnostic and either skip the finding or mark the run operationally failed. - -### 7. Cost And Work Are Not Bounded Enough - -Evidence: - -- `src/providers/vercel-ai-provider.ts:161` defaults agent loops to 1000 steps. -- `src/agent/executor.ts:577` allows repeated nested `lint` calls. -- `src/evaluators/base-evaluator.ts:213` runs one model call per chunk. -- `src/agent/executor.ts:680` and `src/agent/executor.ts:715` glob before applying result caps. - -Impact: - -A single review can multiply cost through agent steps, nested lint calls, rules, chunks, and workspace search. Even a constrained reader executor will be slower than a direct call, so `auto` must have clear thresholds and budgets. - -Recommendation: - -Add explicit budgets: - -- Max target bytes. -- Max caller context bytes. -- Max chunks per rule. -- Max model calls per review. -- Max findings per rule. -- Max wall-clock duration. -- Max headless executor retries. - -### 8. Documentation Still Points At The Old Product - -Evidence: - -- `README.md:148` documents autonomous workspace-agent mode. -- `docs/specs/2026-03-17-agentic-capabilities-design.md:24` frames the roadmap around cross-document agent mode. -- The same spec describes implementation details that are not true now: one agent per rule, final `AgentFindingSchema`, AbortSignal propagation, ripgrep JSON, and Vale JSON omission of agent findings. - -Impact: - -Future contributors will optimize toward the wrong architecture and trust contracts that the code does not implement. - -Recommendation: - -Mark the agentic spec superseded. Replace it with a harness architecture spec that defines the review request, context boundary, executor interface, output contract, and migration plan. - -### 9. Secrets And Sensitive Content Need Better Handling - -Evidence: - -- `src/config/global-config.ts:13` says the global config stores API keys. -- `src/config/global-config.ts:76` creates the config directory with default permissions. -- `src/config/global-config.ts:81` writes the config file with default permissions. -- `src/observability/langfuse-observability.ts:66` records inputs. -- `src/observability/langfuse-observability.ts:67` records outputs. -- `src/providers/vercel-ai-provider.ts:59` can log full prompts and content. - -Impact: - -Provider keys, unreleased docs, caller context, and model outputs can leak through filesystem permissions, telemetry, debug logs, or persisted artifacts. - -Recommendation: - -Use private file modes for config and review artifacts. Make payload telemetry a separate opt-in from metadata telemetry. Add redaction or safe debug modes. - -## Verification Results - -Commands run from `/Users/klinsmann/Projects/TinyRocketLabs/vectorlint`: - -```bash -npm run lint -npm run test:run -npx tsc --noEmit -``` - -Results: - -- `npm run lint` failed before linting because typed ESLint rules were applied to a `.cjs` research script without parser type information. -- `npm run test:run` passed most tests but failed 4 suites during module resolution for `ora` and `@langfuse/otel`. -- `npx tsc --noEmit` failed with contract errors in agent, orchestrator, observability, logging, and provider modules. - -## Recommended Refactor Sequence - -### Phase 1: Stop The Bleeding - -1. Remove or hide the current autonomous workspace-agent mode. -2. Fix `tsc --noEmit`. -3. Fix test module resolution. -4. Fix lint configuration for non-TypeScript research scripts. -5. Add `typecheck` to CI and build validation. - -### Phase 2: Define The Harness Contract - -1. Create a neutral review-domain module. -2. Define `ReviewRequest`, `ReviewRule`, `ReviewContext`, `ReviewBudget`, `ReviewFinding`, `ReviewScore`, and `ReviewDiagnostic`. -3. Define target/context boundary rules. -4. Define structured output shape once. -5. Define execution strategy: `direct`, `reader`, and `auto`. - -### Phase 3: Share Result Projection - -1. Extract evidence location and verification. -2. Extract filtering. -3. Extract severity and scoring. -4. Extract output formatter inputs. -5. Make standard mode and any executor mode use the same projection path. - -### Phase 4: Replace The Agent Loop With Executors - -1. Keep API model execution as direct execution. -2. Add target-scoped reader execution as another executor. -3. Add headless CLI execution behind the same review contract if still useful. -4. Pass the same review request to each executor. -5. Remove model-controlled rule overrides. -6. Remove workspace read/search tools from core review. - -### Phase 5: Update Documentation - -1. Mark the agentic capabilities spec superseded. -2. Write a new harness architecture spec. -3. Update README and CLI docs. -4. Document migration from autonomous workspace-agent mode to direct/reader/auto execution. - -## Suggested Target Architecture - -```text -CLI / API / External Agent - -> ReviewRequestBuilder - -> ReviewExecutor - -> ApiModelExecutor - -> ReaderExecutor - -> HeadlessCliExecutor - -> ResultProjection - -> EvidenceVerifier - -> ViolationFilter - -> ScoreCalculator - -> Diagnostics - -> OutputFormatter - -> line - -> json - -> rdjson - -> vale-json -``` - -The important inversion is that VectorLint no longer asks, "What tools should this agent use?" It asks, "Given this target, these rules, this context, this execution strategy, and this budget, what findings can be verified?" - -## Final Recommendation - -The decided direction is cleaner than the current architecture. The current pain is not a sign that VectorLint is too complex; it is a sign that the codebase is carrying two ownership models and several duplicated runtime contracts. - -Make VectorLint the harness. Let external agents be agents. Keep reader execution only as a bounded way to manage target-content context. - ---- - -## Appendix: Phase 1 — Completed (2026-07-13) - -Phase 1 of the recommended refactor sequence ("Stop The Bleeding") landed on branch `codex/ci/harness-stop-the-bleeding`. It shifts the verification baseline recorded in "Verification Results" above and addresses Finding 1 at the CLI boundary. - -**Shifted baseline.** The 2026-07-13 pre-work capture differed from the original verification results: `npm run lint` and `npm run test:run` already passed, so the earlier lint failure and the four `ora` / `@langfuse/otel` module-resolution failures were stale. Only `npx tsc --noEmit` was still failing. Phase 1 cleared those remaining type errors (narrowed at boundaries; no strict compiler options relaxed) and added durable gates so the baseline cannot silently regress: - -- `npm run typecheck` (`tsc --noEmit`) and `npm run verify` (typecheck + lint + test:run) scripts. -- `vitest.config.ts` for stable test module resolution. -- `docs/research/**` excluded from linting. -- `.github/workflows/typecheck.yml` and `.github/workflows/test.yml` aligned to Node 20. - -**Current state.** `npm run verify`, `npm run build`, and built-CLI standard review smoke runs are green. Per Finding 1, the autonomous workspace-agent surface is deprecated at the CLI boundary: `--mode agent` emits a deprecation warning and falls back to standard evaluation. `src/agent/*` and the agent-mode helpers are retained as compile-only quarantine, unreachable from the CLI, pending removal in Phase 4. The deprecation notice and pointer to this audit live in the README `## Agent Mode` section and the `--mode` help text. - -This appendix records completion only. Findings 2–9 and the proposed core contract remain owned by Phases 2–5; no spec or architecture document is superseded here. diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 50dbffc8..b8586407 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -82,7 +82,7 @@ export function registerMainCommand(program: Command): void { .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') .option('--debug-json', 'Write debug JSON artifacts (raw model output + filter decisions)') .option('--output ', `Output format: ${OUTPUT_FORMATS.join(', ')}`, OUTPUT_FORMATS[0]) - .option('--mode ', 'Execution mode: standard (default). "agent" is deprecated and falls back to standard.', DEFAULT_REVIEW_MODE) + .option('--mode ', 'Execution mode: standard (default). "agent" is unreleased/internal and falls back to standard.', DEFAULT_REVIEW_MODE) .option('-p, --print', 'Suppress interactive progress output in agent mode') .option('--config ', `Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) .argument('[paths...]', 'files or directories to check (required)') diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index f8df367a..1fe29707 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -1168,8 +1168,8 @@ async function buildAgentRuleScores( return results; } -// Retained in quarantine: unreachable from the CLI after --mode agent was -// deprecated (now falls back to standard evaluation). Removed in Phase 4. +// Retained in quarantine: the unreleased --mode agent path now falls back to +// standard evaluation. Removed in Phase 4. // eslint-disable-next-line @typescript-eslint/no-unused-vars -- intentional quarantine async function evaluateFilesInAgentMode( targets: string[], @@ -1326,8 +1326,8 @@ export async function evaluateFiles( if (mode === AGENT_REVIEW_MODE) { options.logger?.warn( - '--mode agent is deprecated and now falls back to standard mode. ' + - 'See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md.', + '--mode agent is an unreleased internal path and now falls back to standard mode. ' + + 'This internal path is being replaced by the bounded harness model.', ); // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index 0b61ea63..98346e42 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -54,11 +54,11 @@ function makeStandardProvider(): StandardProviderSpies { return { provider, runPromptStructured, runAgentToolLoop }; } -// `--mode agent` is deprecated. These tests prove the CLI/evaluateFiles path no +// `--mode agent` is an internal fallback. These tests prove the CLI/evaluateFiles path no // longer reaches the autonomous agent executor: it warns through the injected // logger and falls back to standard evaluation. The retained agent executor // code is covered directly by tests/agent/* and removed in Phase 4. -describe('agent mode deprecation', () => { +describe('agent mode internal fallback', () => { const tempRepos: string[] = []; function createTempRepo(): string { @@ -80,7 +80,7 @@ describe('agent mode deprecation', () => { } }); - it('emits a deprecation warning through the injected logger and falls back to standard evaluation', async () => { + it('emits an internal-rework notice through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); @@ -102,10 +102,10 @@ describe('agent mode deprecation', () => { }); expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('deprecated'), + expect.stringContaining('unreleased internal path'), ); expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md'), + expect.stringContaining('bounded harness model'), ); // Standard evaluation ran; the autonomous executor did not. expect(runAgentToolLoop).not.toHaveBeenCalled(); @@ -134,7 +134,7 @@ describe('agent mode deprecation', () => { logger, }); - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('deprecated')); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('unreleased internal path')); expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(runPromptStructured).toHaveBeenCalled(); }); @@ -158,7 +158,7 @@ describe('agent mode deprecation', () => { scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], }); - // No deprecation warning and no executor invocation in standard mode. + // No internal-rework notice and no executor invocation in standard mode. expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(result.totalFiles).toBe(1); }); From 4c9519de557ea0774a3a20ba33f3ab2a23e2d7c0 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Tue, 14 Jul 2026 15:52:14 +0100 Subject: [PATCH 22/24] Remove planning artifact references from review docs - Replace audit and phase references with stable review contract wording\n- Preserve review behavior and integration semantics\n- Validate review tests and repository verification --- src/review/README.md | 9 ++++----- src/review/boundary.ts | 7 +++---- src/review/budget.ts | 2 +- src/review/executor.ts | 4 ++-- src/review/index.ts | 2 +- src/review/request-builder.ts | 4 ++-- src/review/schemas.ts | 4 ++-- src/review/types.ts | 17 +++++++---------- 8 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/review/README.md b/src/review/README.md index bc2a120b..2b4b461e 100644 --- a/src/review/README.md +++ b/src/review/README.md @@ -19,7 +19,7 @@ A caller builds a `ReviewRequest`; an executor returns a `ReviewResult`. - `request-builder.ts` — `buildReviewRequest()` bridging `PromptFile` to `ReviewRequest`. -## On-page boundary (audit Finding #5) +## On-page boundary - Target content is always in scope. - Caller-supplied context is in scope. @@ -36,7 +36,6 @@ picks `single` for normal-sized inputs and `agent` for large ones. ## Wiring status -Phase 2 (this module) is purely additive and is **not** wired into the CLI. -Phase 3 emits `ReviewResult` via shared finding processing; Phase 4 implements -executors behind `ReviewExecutor` and removes the old agent loop. See -`docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md`. +This module is additive and is **not** wired into the CLI yet. Future CLI +wiring can emit `ReviewResult` through shared finding processing and implement +executors behind `ReviewExecutor`. diff --git a/src/review/boundary.ts b/src/review/boundary.ts index 8dc5bb0c..97e4c0e2 100644 --- a/src/review/boundary.ts +++ b/src/review/boundary.ts @@ -4,9 +4,8 @@ const FILE_URI_RE = /^file:\/\/(?[^/]*)(?\/.*)?$/; /** * Lexically resolves '.' and '..' segments in a path string. Pure: performs no - * filesystem reads. The normalization philosophy is adapted from - * src/agent/path-utils.ts (resolveWithinRoot), which is removed in Phase 4; - * copied here so src/review/ has no agent import and no filesystem access. + * filesystem reads. The helper is kept local so src/review/ has no agent + * import and no filesystem access. * * Symlink canonicalization is intentionally NOT performed inside the contract: * callers are responsible for canonicalizing real files into target/context @@ -41,7 +40,7 @@ export function normalizeReviewUri(uri: string): string { } /** - * Builds the on-page boundary scope (audit Finding #5) from the target URI and + * Builds the on-page boundary scope from the target URI and * any caller-supplied context URIs. Only these URIs are in scope; arbitrary * workspace files are out of scope unless the caller explicitly includes them. */ diff --git a/src/review/budget.ts b/src/review/budget.ts index e7d4cd8b..7f9ca5b8 100644 --- a/src/review/budget.ts +++ b/src/review/budget.ts @@ -3,7 +3,7 @@ import type { ReviewBudget } from './types'; import { VectorlintError } from '../errors'; /** - * Sensible default bounds for a single review (audit Finding #7). Frozen so a + * Sensible default bounds for a single review. Frozen so a * shared default object cannot be accidentally mutated by a caller. * * Note: there is intentionally no finding cap (`maxFindingsPerRule`) — diff --git a/src/review/executor.ts b/src/review/executor.ts index 2eef9011..74ae0100 100644 --- a/src/review/executor.ts +++ b/src/review/executor.ts @@ -15,7 +15,7 @@ export const REVIEW_MODEL_CALLS = ['single', 'agent', 'auto'] as const satisfies /** * The stable domain-level interface every executor implements. Single and - * agent executors are implementations behind this contract (audit Finding #2). + * agent executors are implementations behind this contract. * A headless/autonomous workspace executor is explicitly NOT part of this * contract. */ @@ -25,7 +25,7 @@ export interface ReviewExecutor { /** * Agent-call capability: a bounded way to page through target content for - * context management. NOT a workspace tool (audit Product Decision). An agent + * context management. NOT a workspace tool. An agent * executor receives this capability scoped to the target only. */ export interface ReviewTargetReadCapability { diff --git a/src/review/index.ts b/src/review/index.ts index c44dbe66..e2194b52 100644 --- a/src/review/index.ts +++ b/src/review/index.ts @@ -5,7 +5,7 @@ * source-backed rules flows through this module. The CLI, headless adapters, * and external callers all build a ReviewRequest and receive a ReviewResult. * - * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. + * See README.md for the contract overview. */ export * from './types'; export * from './schemas'; diff --git a/src/review/request-builder.ts b/src/review/request-builder.ts index 72500596..770e33c0 100644 --- a/src/review/request-builder.ts +++ b/src/review/request-builder.ts @@ -63,8 +63,8 @@ function toReviewRule(prompt: PromptFile): ReviewRule { /** * Builds a {@link ReviewRequest} from existing {@link PromptFile}s plus a - * target. This is the conservative bridge Phase 4 uses to convert the current - * prompt pipeline into the review contract without changing how prompts load. + * target. Callers can use this bridge to convert the current prompt pipeline + * into the review contract without changing how prompts load. * Throws a {@link ValidationError} when no prompts are supplied. */ export function buildReviewRequest(params: BuildReviewRequestParams): ReviewRequest { diff --git a/src/review/schemas.ts b/src/review/schemas.ts index a1132cd3..2bca850b 100644 --- a/src/review/schemas.ts +++ b/src/review/schemas.ts @@ -5,8 +5,8 @@ import { REVIEW_BUDGET_SCHEMA } from './budget'; * Zod schemas mirroring src/review/types.ts (boundary validation). * * Every external review-domain shape has a paired schema here so callers and - * external adapters can validate untrusted input at the system boundary - * (audit Finding #3). Schemas are intentionally strict: unknown keys are + * external adapters can validate untrusted input at the system boundary. + * Schemas are intentionally strict: unknown keys are * rejected so legacy scoring-mode, rubric, and model-authored rule-override * fields cannot leak into the new contract. */ diff --git a/src/review/types.ts b/src/review/types.ts index 97e22039..9750d530 100644 --- a/src/review/types.ts +++ b/src/review/types.ts @@ -5,8 +5,6 @@ * source-backed rules flows through this module. Callers build a * {@link ReviewRequest} and executors return a {@link ReviewResult}. * - * See docs/audits/2026-07-10-vectorlint-harness-architecture-audit.md. - * * This module is implementation-neutral: it deliberately exposes no legacy * scoring-mode, rubric, or model-authored rule-override surface. `modelCall` * selects how the reviewer model is invoked, not how rules are scored. @@ -22,8 +20,8 @@ export interface ReviewViolationCondition { } /** - * Target content under review. The on-page boundary (audit Finding #5) is - * enforced against this: executors may only read sections of `content`. + * Target content under review. The on-page boundary is enforced against this: + * executors may only read sections of `content`. */ export interface ReviewTarget { /** Stable absolute URI (file:// or virtual scheme for in-memory content). */ @@ -38,7 +36,7 @@ export interface ReviewTarget { /** * A source-backed, caller-authored rule. Model-authored rule overrides are - * explicitly disallowed (audit Finding #1, #5). + * explicitly disallowed. */ export interface ReviewRule { /** Stable id, formatted Pack.Rule[.Criterion] in output. */ @@ -56,7 +54,7 @@ export interface ReviewRule { } /** - * Caller-supplied context. Explicitly in scope (audit boundary rule): + * Caller-supplied context. Explicitly in scope: * VectorLint does NOT discover workspace files as context on its own; the * caller owns exploration and context gathering. */ @@ -82,7 +80,7 @@ export interface ReviewScope { } /** - * Hard bounds on a single review (audit Finding #7). These limit work, not + * Hard bounds on a single review. These limit work, not * output. Executors MUST check these and surface a ReviewDiagnostic (or fail * the run) when exceeded. */ @@ -97,9 +95,8 @@ export interface ReviewBudget { /** * Controls optional output/telemetry behavior. Diagnostics are always part of - * ReviewResult; this policy does not hide operational warnings. Audit - * Finding #9: payload telemetry must be a separate opt-in from metadata - * telemetry. + * ReviewResult; this policy does not hide operational warnings. Payload + * telemetry is a separate opt-in from metadata telemetry. */ export interface ReviewOutputPolicy { /** Include usage/cost in the result. */ From af25452e2752e0ebb7d83b04d71df1963ecacd6e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Thu, 23 Jul 2026 00:50:02 +0100 Subject: [PATCH 23/24] refactor(vectorlint): remove evaluation modes - Make violation finding and density scoring the sole evaluation path - Ignore legacy type frontmatter and remove mode-specific rule metadata - Remove rubric documentation, planning artifacts, and obsolete tests --- AGENTS.md | 4 +- CONFIGURATION.md | 2 +- CREATING_RULES.md | 32 +-- README.md | 2 +- docs/best-practices.mdx | 18 -- docs/customize-style-rules.mdx | 18 +- docs/docs.json | 3 +- docs/false-positive-tuning.mdx | 2 +- docs/frontmatter-fields.mdx | 3 - docs/how-it-works.mdx | 19 +- docs/initial-style-check.mdx | 4 +- docs/introduction.mdx | 8 +- docs/project-config.mdx | 2 +- docs/rubric-scoring.mdx | 8 - docs/style-guide.mdx | 16 +- docs/use-cases.mdx | 4 +- presets/VectorLint/ai-pattern.md | 1 - presets/VectorLint/capitalization.md | 3 +- presets/VectorLint/consistency.md | 1 - presets/VectorLint/inclusivity.md | 1 - presets/VectorLint/passive-voice.md | 3 +- presets/VectorLint/repetition.md | 3 +- presets/VectorLint/unsupported-claims.md | 3 +- presets/VectorLint/wordiness.md | 3 +- src/agent/executor.ts | 10 +- src/chunking/merger.ts | 6 +- src/cli/orchestrator.ts | 217 +++++++---------- src/config/constants.ts | 2 +- src/debug/run-artifact.ts | 1 - src/evaluators/accuracy-evaluator.ts | 8 +- src/evaluators/base-evaluator.ts | 145 +----------- src/evaluators/types.ts | 5 - src/findings/finding-evidence-verifier.ts | 14 +- src/findings/index.ts | 14 +- src/findings/processor.ts | 29 +-- src/findings/scorer.ts | 38 +-- src/findings/severity.ts | 20 +- src/findings/types.ts | 44 +--- src/output/json-formatter.ts | 2 - src/prompts/prompt-validator.ts | 20 -- src/prompts/schema.ts | 220 +----------------- src/prompts/target.ts | 14 +- src/review/schemas.ts | 1 - src/review/types.ts | 1 - src/schemas/prompt-schemas.ts | 47 +--- src/scoring/index.ts | 8 +- src/scoring/scorer.ts | 213 +---------------- tests/agent/agent-executor.test.ts | 98 -------- tests/evaluations/test-rules/Test/clarity.md | 1 - .../test-rules/Test/consistency.md | 1 - .../test-rules/Test/passive-voice.md | 1 - .../test-rules/Test/readability.md | 1 - .../evaluations/test-rules/Test/wordiness.md | 1 - tests/evaluator.test.ts | 4 +- tests/findings/README.md | 83 ------- .../finding-evidence-verifier.test.ts | 18 +- tests/findings/module-surface.test.ts | 2 +- tests/findings/processor.test.ts | 38 ++- tests/findings/scorer.test.ts | 29 +-- tests/findings/severity.test.ts | 13 +- tests/findings/types.test.ts | 56 +---- .../capitalization-rules/capitalization.md | 3 +- .../rules/consistency-rules/consistency.md | 1 - .../rules/inclusivity-rules/inclusivity.md | 1 - .../passive-voice-rules/passive-voice.md | 3 +- .../rules/repetition-rules/repetition.md | 3 +- .../unsupported-claims.md | 3 +- .../rules/wordiness-rules/wordiness.md | 3 +- tests/orchestrator-agent-output.test.ts | 13 +- tests/orchestrator-filtering.test.ts | 160 +++---------- tests/orchestrator-finding-criteria.test.ts | 126 ---------- ...=> orchestrator-finding-processor.test.ts} | 82 +++---- tests/prompt-loader-validation.test.ts | 68 +----- tests/prompt-schema.test.ts | 58 +---- tests/review/request-builder.test.ts | 1 - tests/review/types.test.ts | 2 +- tests/scoring-types.test.ts | 101 +------- tests/target.test.ts | 42 +--- tests/validator.test.ts | 27 +-- 79 files changed, 337 insertions(+), 1948 deletions(-) delete mode 100644 docs/rubric-scoring.mdx delete mode 100644 tests/findings/README.md delete mode 100644 tests/orchestrator-finding-criteria.test.ts rename tests/{orchestrator-check-processor.test.ts => orchestrator-finding-processor.test.ts} (77%) diff --git a/AGENTS.md b/AGENTS.md index 77d0a7ac..0373fad3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,7 +17,7 @@ This repository implements VectorLint — a prompt‑driven, structured‑output - `providers/` — LLM abstractions (OpenAI, Anthropic, Azure, Gemini), request builder, provider factory - `scan/` — file discovery (fast‑glob) honoring config and exclusions - `schemas/` — Zod schemas for all external data (API responses, config, CLI, env) - - `scoring/` — score calculation (density-based for check, rubric-based for judge) + - `scoring/` — density-based score calculation - `types/` — TypeScript type definitions - `presets/` — bundled rule packs (e.g., `VectorLint/`) - `tests/` — Vitest specs for config, scanning, evaluation, providers @@ -182,7 +182,7 @@ For documents >600 words, VectorLint automatically chunks content: - Type safety: strict TypeScript with no `any`; use `unknown` + schema validation for external data - Dependency inversion: depend on `LLMProvider` and `SearchProvider` interfaces; keep providers thin (transport only) - Dependency injection: inject `RequestBuilder` via provider constructor to avoid coupling -- Separation of concerns: rules define rubric; schemas enforce structure; CLI orchestrates; evaluators process; reporters format +- Separation of concerns: rules define criteria; schemas enforce structure; CLI orchestrates; evaluators process; reporters format - Separation of concerns: when a file starts combining contracts, orchestration, and utility logic, extract shared helpers and types into focused modules - Extensibility: add providers by implementing `LLMProvider` or `SearchProvider`; add evaluators via registry pattern - Shared domain constants: avoid magic strings for core runtime concepts; define shared constants, enums, or types and import them where needed diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 34dfcf8c..ea7f8506 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -128,7 +128,7 @@ CONFIDENCE_THRESHOLD=0.75 ``` - Default: `0.75` -- Applies to surfaced violations in check and judge evaluations +- Applies to surfaced violations - Invalid values gracefully fall back to the default --- diff --git a/CREATING_RULES.md b/CREATING_RULES.md index 7b800c9d..7254cd25 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -6,8 +6,7 @@ A comprehensive guide to creating powerful, reusable content evaluations using V - [Overview](#overview) - [Rule Anatomy](#rule-anatomy) -- [Evaluation Modes](#evaluation-modes) -- [Check Rules](#check-rules) +- [How Rules Work](#how-rules-work) - [Target Specification](#target-specification) - [Configuration Reference](#configuration-reference) - [Best Practices](#best-practices) @@ -40,7 +39,6 @@ Every rule is a Markdown file with two parts: id: MyEval name: My Content Evaluator evaluator: base -type: check severity: error --- @@ -71,34 +69,16 @@ project/ --- -## Evaluation Modes +## How Rules Work -VectorLint uses a single **Base Evaluator** (`evaluator: base`) that operates in one mode, determined by the `type` field: - -| Mode | `type` | Use Case | Scoring | Output | -| ---------- | ------- | ------------------------------------- | --------------------------- | ----------------------- | -| **Check** | `check` | Pass/fail checks, counting violations | 10 points - 1 per violation | List of specific issues | - -The deprecated `type: judge` (alias `subjective`) is no longer supported; a rule using it is rejected at load. - -### When to Use Check - -- You need to find specific errors (e.g., "Find all grammar mistakes") -- The check is binary (Pass/Fail) for each item -- You want a list of specific violations to fix - ---- - -## Check Rules - -Check rules are perfect for finding specific issues. The LLM lists violations, and the score is calculated based on the count of violations. +The LLM lists specific violations, and VectorLint calculates a density-based +score from the verified findings. ### Minimal Example ```markdown --- evaluator: base -type: check id: GrammarChecker name: Grammar Checker severity: error @@ -174,7 +154,6 @@ target: | ------------- | ------------- | -------- | -------------------------------------------------------------- | | `specVersion` | string/number | No | Rule specification version (use `1.0.0`) | | `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | -| `type` | string | No | Mode: `check` (default: `check`). `judge`/`subjective` rejected at load | | `id` | string | **Yes** | Unique identifier (used in error reporting) | | `name` | string | **Yes** | Human-readable name | @@ -225,12 +204,11 @@ Help the LLM understand your domain: ## Examples -### Example 1: Simple Grammar Check (Check) +### Example 1: Simple Grammar Rule ```markdown --- evaluator: base -type: check id: GrammarChecker name: Grammar Checker severity: error diff --git a/README.md b/README.md index 5657ead0..bdbef88b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # VectorLint: Prompt it, Lint it! [![npm version](https://img.shields.io/npm/v/vectorlint.svg)](https://www.npmjs.com/package/vectorlint) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -VectorLint is a command-line tool that evaluates and scores content using LLMs. It uses [LLM-as-a-Judge](https://en.wikipedia.org/wiki/LLM-as-a-Judge) to catch terminology, technical accuracy, and style issues that require contextual understanding. +VectorLint is a command-line tool that uses LLMs to find and score terminology, technical accuracy, and style issues that require contextual understanding. ![VectorLint Screenshot](./assets/VectorLint_screenshot.jpeg) diff --git a/docs/best-practices.mdx b/docs/best-practices.mdx index 05c6ad84..9eab45e4 100644 --- a/docs/best-practices.mdx +++ b/docs/best-practices.mdx @@ -51,24 +51,6 @@ LLMs evaluate content relative to an implied standard. Make that standard explic A grammar rule without context produces generic grammar findings. The same rule with a developer audience context produces findings calibrated to technical writing conventions. -## Use weights that reflect real priorities - -In judge rules, weights are the single most important configuration decision. They determine which criteria actually control the final score. Treat them as a statement of your content team's values — not arbitrary numbers. - -```yaml -criteria: - - name: Technical Accuracy - weight: 40 # Factual errors erode user trust — this matters most - - name: Clarity - weight: 30 # Unclear docs generate support tickets - - name: Tone - weight: 20 # Important but recoverable in editing - - name: SEO - weight: 10 # Nice to have, never at the expense of the above -``` - -If everything has the same weight, nothing is prioritized. - ## Tier strictness by content type Not all content deserves the same quality bar. Apply strictness in proportion to how much a failure costs — measured in user trust, support load, or brand impact. diff --git a/docs/customize-style-rules.mdx b/docs/customize-style-rules.mdx index bc39261e..e19760c4 100644 --- a/docs/customize-style-rules.mdx +++ b/docs/customize-style-rules.mdx @@ -28,7 +28,6 @@ Create a new file at `.github/rules/MyTeam/grammar-checker.md`: id: GrammarChecker name: Grammar Checker evaluator: base -type: check severity: error --- @@ -102,23 +101,8 @@ You are a headline evaluator for developer blog posts. Assess whether the headli - Direct answers over lengthy preambles ``` -**Use meaningful weights** to reflect real-world importance. Scale them to signal what actually matters in your content workflow: - -```yaml -criteria: - - name: Technical Accuracy - id: TechnicalAccuracy - weight: 40 # Critical — factual errors erode trust - - name: Readability - id: Readability - weight: 30 # Important — but recoverable in editing - - name: SEO - id: SEO - weight: 10 # Nice to have -``` - ## Next steps -- [Defining your style rules](/style-guide) — full reference for rule anatomy, evaluation modes, targeting, and examples +- [Defining your style rules](/style-guide) — full reference for rule anatomy, targeting, and examples - [Project Configuration](/project-config) — control which rules run on which files - [LLM Providers](/llm-providers) — switch providers or tune false-positive filtering diff --git a/docs/docs.json b/docs/docs.json index 5b7727c8..a4f8e344 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -46,8 +46,7 @@ "group": "Quality scoring", "pages": [ "quality-scoring", - "density-scoring", - "rubric-scoring" + "density-scoring" ] }, { diff --git a/docs/false-positive-tuning.mdx b/docs/false-positive-tuning.mdx index 299d1122..50e00f0e 100644 --- a/docs/false-positive-tuning.mdx +++ b/docs/false-positive-tuning.mdx @@ -6,7 +6,7 @@ description: Use `CONFIDENCE_THRESHOLD` and strictness overrides together to red VectorLint gives you two complementary levers for controlling evaluation precision. Used together, they let you calibrate how aggressively VectorLint surfaces findings — globally across your project, and per content type. - **`CONFIDENCE_THRESHOLD`** — controls how strictly the PAT pipeline filters raw model candidates before surfacing them. A global setting that applies to every evaluation. -- **Strictness overrides** — controls how harshly check rules score error density for specific file patterns. Set per content type in `.vectorlint.ini`. +- **Strictness overrides** — controls how harshly rules score error density for specific file patterns. Set per content type in `.vectorlint.ini`. Understanding when to reach for each one is the key to a low-noise, high-signal workflow. diff --git a/docs/frontmatter-fields.mdx b/docs/frontmatter-fields.mdx index d840d7b9..31ea9267 100644 --- a/docs/frontmatter-fields.mdx +++ b/docs/frontmatter-fields.mdx @@ -9,7 +9,6 @@ Every VectorLint rule file begins with a YAML frontmatter block that controls ho --- id: MyRule name: My Rule -type: check severity: error --- @@ -29,7 +28,6 @@ Your LLM prompt goes here. |---|---|---| | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | | `evaluator` | `base` | Evaluator type. Use `base` for standard rules. Use `technical-accuracy` for rules that verify factual claims against live web search (requires a search provider). | -| `type` | `check` | Evaluation mode. `check` finds specific, countable violations and scores them by error density. `judge` (and its deprecated `subjective` alias) is no longer supported — a rule with `type: judge` or `type: subjective` is rejected at load. | | `severity` | `warning` | How a failing result is reported. `error` causes a non-zero exit code; `warning` reports the issue without failing the run. | | `evaluateAs` | `chunk` | Whether to evaluate content in sections (`chunk`) or as a single unit (`document`). Chunking is applied automatically to documents over 600 words. Set to `document` to disable chunking for a specific rule. | | `target` | _(none)_ | Regex specification to extract and evaluate a specific portion of the content, such as the H1 headline. See [Target fields](#target-fields) below. | @@ -63,7 +61,6 @@ The following rule uses the most commonly-used fields. It targets the H1 headlin ```markdown --- evaluator: base -type: check id: HeadlineQuality name: Headline Quality severity: error diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index 93e42fcf..dd1bc7c9 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -1,9 +1,9 @@ --- title: How it works -description: How VectorLint evaluates content using LLM-as-a-Judge, filtering, and two scoring methods. +description: How VectorLint evaluates content, filters findings, and calculates scores. --- -VectorLint evaluates your content by sending it to a large language model (LLM) along with a structured prompt that defines your quality criteria. The LLM acts as a judge — reading your content, applying the criteria, and returning findings. VectorLint then filters those findings through a deterministic pipeline before surfacing violations in the CLI. +VectorLint sends your content to a large language model (LLM) with a structured prompt that defines your quality criteria. The model returns candidate findings, which VectorLint filters through a deterministic pipeline before surfacing violations in the CLI. If you haven't run your first check yet, start with the [Quickstart](/quickstart). This page is for understanding the architecture behind what you're running. @@ -19,10 +19,7 @@ If no `.vectorlint.ini` exists, VectorLint detects `VECTORLINT.md` and creates a ### 2. LLM evaluation -VectorLint sends your content to the configured LLM provider with each rule's prompt. Depending on the rule type, the model returns either: - -- **A list of specific violations** (check rules) — for countable errors like grammar mistakes or banned terms -- **A scored rubric** (judge rules) — for quality dimensions like tone, clarity, or technical accuracy rated on a 1–4 scale +VectorLint sends your content to the configured LLM provider with each rule's prompt. The model returns a list of specific violations with supporting evidence. Rules run concurrently up to the `Concurrency` limit set in `.vectorlint.ini`. @@ -39,11 +36,7 @@ You can tune how aggressively the filter operates with `CONFIDENCE_THRESHOLD` (d ## Scoring -VectorLint uses two scoring methods depending on the rule type. - -**Density-based scoring** (check rules) calculates error density, violations per 100 words. VectorLint weights a single error in a short document more heavily than the same error in a long one. This normalizes quality assessment across documents of any length. - -**Rubric-based scoring** (judge rules) normalizes the LLM's 1–4 rating to a 1–10 scale and computes a weighted average across all criteria. Criteria weights reflect your real-world priorities — technical accuracy might carry a weight of 40 while SEO carries a weight of 10. +VectorLint calculates error density from verified violations per 100 words. A single error in a short document weighs more heavily than the same error in a long one, which normalizes quality assessment across documents of different lengths. See [Quality scoring](/quality-scoring) for the full scoring reference. @@ -52,7 +45,7 @@ See [Quality scoring](/quality-scoring) for the full scoring reference. VectorLint gives you two complementary tools for expressing your content standards: - **`VECTORLINT.md`** — plain-language style instructions that apply globally to every evaluation. The fastest path to useful output: no rule files, no configuration syntax, just plain English instructions that the LLM uses as context for every check. -- **Rule pack files** — structured LLM prompts for specific, measurable checks with weighted scoring criteria. Use these when you need reproducible, auditable results on a particular dimension of quality. +- **Rule pack files** — structured LLM prompts for specific, measurable standards. Use these when you need reproducible, auditable results on a particular dimension of quality. The two work together: `VECTORLINT.md` sets the baseline context, and rule pack files enforce specific criteria on top of it. @@ -62,4 +55,4 @@ See [Defining your style rules](/style-guide) for how to create both. - [Quickstart](/quickstart) — run your first evaluation in minutes - [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules -- [Configuring a project](/project-config) — map files to rule packs \ No newline at end of file +- [Configuring a project](/project-config) — map files to rule packs diff --git a/docs/initial-style-check.mdx b/docs/initial-style-check.mdx index 2292cac6..ededf0fe 100644 --- a/docs/initial-style-check.mdx +++ b/docs/initial-style-check.mdx @@ -65,9 +65,9 @@ Because `VECTORLINT.md` is plain language, anyone on your team can read and edit `VECTORLINT.md` handles broad, always-applicable standards well. It also supports creating custom rule pack files — targeted LLM prompts that enforce specific, measurable criteria that plain-language instructions can't capture precisely enough. -For example, a `VECTORLINT.md` rule like "avoid passive voice" gives the LLM general guidance. A rule pack file with a structured prompt, scoring rubric, and audience context gives it precise, reproducible criteria with weighted scoring. +For example, a `VECTORLINT.md` rule like "avoid passive voice" gives the LLM general guidance. A rule pack file with a structured prompt and audience context gives it precise, reproducible criteria. When you're ready to go beyond `VECTORLINT.md`: - [Customize style rules](/customize-style-rules) — step-by-step guide to writing your first rule pack file and crafting effective prompts -- [Defining your style rules](/style-guide) — full reference for rule anatomy, evaluation modes, targeting, and examples +- [Defining your style rules](/style-guide) — full reference for rule anatomy, targeting, and examples diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 02197adf..1041f912 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -5,7 +5,7 @@ description: "VectorLint is a large language model (LLM)-based prose linter that ## What is VectorLint? -VectorLint is a command-line tool that evaluates and scores documentation using large language models (LLMs). Instead of regex patterns that can only catch surface-level issues, VectorLint uses an [LLM-as-a-Judge](https://en.wikipedia.org/wiki/LLM-as-a-Judge) approach to catch terminology misuse, technical inaccuracies, and style inconsistencies that require contextual understanding to detect. +VectorLint is a command-line tool that evaluates and scores documentation using large language models (LLMs). Unlike regex patterns that catch only surface-level issues, VectorLint can find terminology misuse, technical inaccuracies, and style inconsistencies that require contextual understanding. If you can write a prompt for it, you can lint it with VectorLint. @@ -36,11 +36,7 @@ This gives documentation teams something they haven't had before: a **shared, me ## How scoring works -VectorLint uses two scoring methods depending on the rule type: - -**Density-based scoring** is used for rules that count discrete violations (like a grammar checker). VectorLint calculates scores based on error density — errors per 100 words — so results are comparable across documents of any length. - -**Rubric-based scoring** is used for rules that measure quality on a spectrum (like tone or completeness). The LLM scores each criterion on a 1–4 scale, which VectorLint normalizes to a 1–10 scale for consistent reporting. +VectorLint calculates scores from error density—verified violations per 100 words—so results are comparable across documents of different lengths. ## How false positives are reduced diff --git a/docs/project-config.mdx b/docs/project-config.mdx index e95656b2..e78d00b7 100644 --- a/docs/project-config.mdx +++ b/docs/project-config.mdx @@ -95,7 +95,7 @@ Grammar.strictness=9 ## Strictness Overrides -You can tune how harshly individual check rules score content on a per-pattern basis. +You can tune how harshly individual rules score content on a per-pattern basis. ```ini [pattern] diff --git a/docs/rubric-scoring.mdx b/docs/rubric-scoring.mdx deleted file mode 100644 index b5c52e1a..00000000 --- a/docs/rubric-scoring.mdx +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: Rubric Scoring -description: This page is coming soon. ---- - - - This page is coming soon. Check back for updates, or follow the [VectorLint GitHub repository](https://github.com/TRocket-Labs/vectorlint) for the latest news. - diff --git a/docs/style-guide.mdx b/docs/style-guide.mdx index 3dfe0c20..bf29b1cd 100644 --- a/docs/style-guide.mdx +++ b/docs/style-guide.mdx @@ -60,7 +60,6 @@ Each rule file has two parts: YAML frontmatter that configures how VectorLint ha id: MyEval name: My Content Evaluator evaluator: base -type: check severity: error --- @@ -79,28 +78,18 @@ Your detailed instructions for the LLM go here. | Field | Default | Description | |-------|---------|-------------| | `evaluator` | `base` | Evaluator type. Use `base` for most rules; `technical-accuracy` for fact-checking rules that need search. | -| `type` | `check` | Evaluation mode. `check` finds specific, countable violations. `judge` (and its `subjective` alias) is no longer supported and is rejected at load. | | `severity` | `warning` | How failures are reported: `error` or `warning`. | | `evaluateAs` | `chunk` | Whether to evaluate content as a whole (`document`) or in sections (`chunk`). | | `target` | _(none)_ | Regex to target a specific part of the content (e.g., only the H1 headline). | | `specVersion` | _(none)_ | Rule spec version. Use `1.0.0`. | -## Evaluation Modes - -VectorLint uses a single evaluation mode, set with the `type` field. - -| Mode | `type` value | Best for | -|------|-------------|----------| -| Check | `check` | Finding specific, countable violations (grammar errors, banned terms) | - -### Check rules +## Finding and scoring violations The LLM returns a list of specific violations. VectorLint scores the result using **error density** (violations per 100 words), so a single error in a short document weighs more than the same error in a long one. ```markdown --- evaluator: base -type: check id: GrammarChecker name: Grammar Checker severity: error @@ -162,12 +151,11 @@ RunRules=Acme ## Examples -### AI Pattern Detector (Check) +### AI Pattern Detector ```markdown --- evaluator: base -type: check id: AIPatterns name: AI Pattern Detector severity: warning diff --git a/docs/use-cases.mdx b/docs/use-cases.mdx index 84bddf70..75091090 100644 --- a/docs/use-cases.mdx +++ b/docs/use-cases.mdx @@ -11,7 +11,7 @@ VectorLint fits anywhere a team publishes written content at scale and needs con **The path:** Write your standards as plain-language instructions in `VECTORLINT.md`. VectorLint converts them into a "Style Guide Compliance" rule and runs it against every file, every time — no regex, no word lists, no manual review for comma usage or passive voice. -If your team has a detailed guide, see [Defining your style rules](/style-guide) for an extraction prompt that converts it into a `VECTORLINT.md`-optimized file. For more targeted enforcement, write rule pack files that score specific criteria on a weighted rubric. +If your team has a detailed guide, see [Defining your style rules](/style-guide) for an extraction prompt that converts it into a `VECTORLINT.md`-optimized file. For more targeted enforcement, write rule pack files that score specific criteria. **Who does this:** Technical writing teams, content operations teams, developer advocacy teams. @@ -88,4 +88,4 @@ See [Configuring a project](/project-config) for the full configuration referenc - [Quickstart](/quickstart) — run your first evaluation in under ten minutes - [How it works](/how-it-works) — understand the evaluation pipeline, PAT filtering, and scoring -- [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules \ No newline at end of file +- [Defining your style rules](/style-guide) — create VECTORLINT.md and write custom rules diff --git a/presets/VectorLint/ai-pattern.md b/presets/VectorLint/ai-pattern.md index 382b8acd..9c0ec2ec 100644 --- a/presets/VectorLint/ai-pattern.md +++ b/presets/VectorLint/ai-pattern.md @@ -1,7 +1,6 @@ --- specVersion: 2.0.0 evaluator: base -type: check id: AIPattern name: AI Pattern severity: warning diff --git a/presets/VectorLint/capitalization.md b/presets/VectorLint/capitalization.md index 6d12fabc..e454d51a 100644 --- a/presets/VectorLint/capitalization.md +++ b/presets/VectorLint/capitalization.md @@ -1,7 +1,6 @@ --- specVersion: 1.0.0 evaluator: base -type: check id: Capitalization name: Capitalization severity: warning @@ -10,4 +9,4 @@ evaluateAs: document # Capitalization -Flag any instance where headings within the same document are inconsistent in their capitalization style. Identify the dominant style used across headings — either title case (e.g., "Getting Started With the API") or sentence case (e.g., "Getting started with the API") — and flag any heading that deviates from it. For title case, also flag when the document mixes AP and Chicago conventions, such as capitalizing prepositions in some headings but not others (e.g., "Working With Your Team" vs. "Working with Your Team"). Do not flag proper nouns or product names that are always capitalized regardless of heading style. \ No newline at end of file +Flag any instance where headings within the same document are inconsistent in their capitalization style. Identify the dominant style used across headings — either title case (e.g., "Getting Started With the API") or sentence case (e.g., "Getting started with the API") — and flag any heading that deviates from it. For title case, also flag when the document mixes AP and Chicago conventions, such as capitalizing prepositions in some headings but not others (e.g., "Working With Your Team" vs. "Working with Your Team"). Do not flag proper nouns or product names that are always capitalized regardless of heading style. diff --git a/presets/VectorLint/consistency.md b/presets/VectorLint/consistency.md index 4be9b45a..fb17a50d 100644 --- a/presets/VectorLint/consistency.md +++ b/presets/VectorLint/consistency.md @@ -1,7 +1,6 @@ --- specVersion: 1.0.0 evaluator: base -type: check id: Consistency name: Consistency severity: warning diff --git a/presets/VectorLint/inclusivity.md b/presets/VectorLint/inclusivity.md index 34f7ed4a..fab769ce 100644 --- a/presets/VectorLint/inclusivity.md +++ b/presets/VectorLint/inclusivity.md @@ -1,7 +1,6 @@ --- specVersion: 1.0.0 evaluator: base -type: check id: Inclusivity name: Inclusivity severity: warning diff --git a/presets/VectorLint/passive-voice.md b/presets/VectorLint/passive-voice.md index 5231941f..e71c93d0 100644 --- a/presets/VectorLint/passive-voice.md +++ b/presets/VectorLint/passive-voice.md @@ -1,7 +1,6 @@ --- specVersion: 1.0.0 evaluator: base -type: check id: PassiveVoice name: Passive Voice severity: warning @@ -12,4 +11,4 @@ evaluateAs: document Flag any sentence that uses passive voice in a way that makes the writing less clear, less direct, or more evasive — for example, when the doer is omitted and that omission reduces accountability or clarity ("Mistakes were made"), or when a modal passive weakens an instruction ("The file should be saved before closing"). -Do not flag passive voice when the sentence is describing a state, condition, or feature rather than an action; when the doer is obvious, irrelevant, or already established in the immediately preceding sentence; or when an active rewrite would be awkward or less natural than the original. \ No newline at end of file +Do not flag passive voice when the sentence is describing a state, condition, or feature rather than an action; when the doer is obvious, irrelevant, or already established in the immediately preceding sentence; or when an active rewrite would be awkward or less natural than the original. diff --git a/presets/VectorLint/repetition.md b/presets/VectorLint/repetition.md index fe87ae79..6849b9c1 100644 --- a/presets/VectorLint/repetition.md +++ b/presets/VectorLint/repetition.md @@ -1,7 +1,6 @@ --- specVersion: 2.0.0 evaluator: base -type: check id: Repetition name: Repetition severity: warning @@ -9,4 +8,4 @@ evaluateAs: document --- # Repetition -Flag any instance where the same word appears consecutively or near-consecutively in a sentence (e.g., "the the build" or "you can can also"), or where the same idea is expressed in more than one sentence within the same section without adding new information (e.g., "This speeds up your workflow. It helps teams work faster and get more done."). Do not flag repeated product names, feature names, or technical terms where varying the language would introduce ambiguity (e.g., "Components" repeated throughout a section about components), and do not flag repetition across structurally independent elements like cards, list items, or callouts that are designed to be read on their own (e.g., a card description that echoes a phrase from the body text it links to). \ No newline at end of file +Flag any instance where the same word appears consecutively or near-consecutively in a sentence (e.g., "the the build" or "you can can also"), or where the same idea is expressed in more than one sentence within the same section without adding new information (e.g., "This speeds up your workflow. It helps teams work faster and get more done."). Do not flag repeated product names, feature names, or technical terms where varying the language would introduce ambiguity (e.g., "Components" repeated throughout a section about components), and do not flag repetition across structurally independent elements like cards, list items, or callouts that are designed to be read on their own (e.g., a card description that echoes a phrase from the body text it links to). diff --git a/presets/VectorLint/unsupported-claims.md b/presets/VectorLint/unsupported-claims.md index 48f3178b..c2146a38 100644 --- a/presets/VectorLint/unsupported-claims.md +++ b/presets/VectorLint/unsupported-claims.md @@ -1,7 +1,6 @@ --- specVersion: 1.0.0 evaluator: base -type: check id: UnsupportedClaims name: Unsupported Claims severity: warning @@ -13,4 +12,4 @@ Flag any claim that states a specific number, percentage, comparison, or causal Also flag vague authority citations where the source is unnamed (e.g., "experts say," "research shows," "it is well established"). -Do not flag claims the author makes about their own product or organization in the first person (e.g., "we processed over 1M requests last month," "our platform supports 40+ integrations"), as these are self-reported and do not require a third-party source. \ No newline at end of file +Do not flag claims the author makes about their own product or organization in the first person (e.g., "we processed over 1M requests last month," "our platform supports 40+ integrations"), as these are self-reported and do not require a third-party source. diff --git a/presets/VectorLint/wordiness.md b/presets/VectorLint/wordiness.md index 3e130014..87d386f3 100644 --- a/presets/VectorLint/wordiness.md +++ b/presets/VectorLint/wordiness.md @@ -1,7 +1,6 @@ --- specVersion: 1.0.0 evaluator: base -type: check id: Wordiness name: Wordiness severity: warning @@ -10,4 +9,4 @@ evaluateAs: document # Wordiness -Flag any instance where a wordy phrase is used in place of a shorter, simpler equivalent (e.g., "in order to" instead of "to", "due to the fact that" instead of "because", "has the ability to" instead of "can", "at this point in time" instead of "at this point"). This includes multi-word phrases that can be reduced to a single word, roundabout constructions that obscure a simpler verb or preposition, and redundant pairs where one word carries the full meaning (e.g., "gather together" instead of "gather", "close down" instead of "close"). Do not flag phrases where the longer form is required for clarity or where the substitution would change the meaning. \ No newline at end of file +Flag any instance where a wordy phrase is used in place of a shorter, simpler equivalent (e.g., "in order to" instead of "to", "due to the fact that" instead of "because", "has the ability to" instead of "can", "at this point in time" instead of "at this point"). This includes multi-word phrases that can be reduced to a single word, roundabout constructions that obscure a simpler verb or preposition, and redundant pairs where one word carries the full meaning (e.g., "gather together" instead of "gather", "close down" instead of "close"). Do not flag phrases where the longer form is required for clarity or where the substitution would change the meaning. diff --git a/src/agent/executor.ts b/src/agent/executor.ts index 68399b1f..eda9f0e4 100644 --- a/src/agent/executor.ts +++ b/src/agent/executor.ts @@ -1,7 +1,7 @@ import { readdir, readFile } from 'fs/promises'; import * as os from 'os'; import fg from 'fast-glob'; -import { buildCheckLLMSchema, isJudgeResult, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; +import { buildEvaluationLLMSchema, type GateChecks, type PromptEvaluationResult } from '../prompts/schema'; import type { PromptFile } from '../prompts/prompt-loader'; import { Type, Severity } from '../evaluators/types'; import { computeFilterDecision } from '../evaluators/violation-filter'; @@ -99,7 +99,7 @@ function buildUnknownRuleSourceError(ruleSource: string, validSources: string[]) } function fallbackMessage(result: PromptEvaluationResult): string { - if (!isJudgeResult(result) && result.reasoning) { + if (result.reasoning) { return result.reasoning; } return 'Potential issue detected'; @@ -594,9 +594,7 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< nestedToolUsage = mergeTokenUsage(nestedToolUsage, result.usage); let findingsRecorded = 0; - const violations = isJudgeResult(result) - ? result.criteria.flatMap((criterion) => criterion.violations) - : result.violations; + const violations = result.violations; for (const violation of violations) { const wasRecorded = await appendInlineFinding({ violation, @@ -616,7 +614,7 @@ export async function runAgentExecutor(params: RunAgentExecutorParams): Promise< return { ok: true, findingsRecorded, - schema: buildCheckLLMSchema().name, + schema: buildEvaluationLLMSchema().name, }; } diff --git a/src/chunking/merger.ts b/src/chunking/merger.ts index d78074d8..c76e17ff 100644 --- a/src/chunking/merger.ts +++ b/src/chunking/merger.ts @@ -1,8 +1,8 @@ -import type { CheckItem } from "../prompts/schema"; +import type { EvaluationItem } from "../prompts/schema"; export function mergeViolations( - chunkViolations: CheckItem[][] -): CheckItem[] { + chunkViolations: EvaluationItem[][] +): EvaluationItem[] { const all = chunkViolations.flat(); // Deduplicate using composite key (quoted_text + description + analysis) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 7e40f5a1..5b36ea40 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -9,7 +9,6 @@ import { ValeJsonFormatter, type JsonIssue } from '../output/vale-json-formatter import { JsonFormatter, type Issue } from '../output/json-formatter'; import { RdJsonFormatter } from '../output/rdjson-formatter'; import { printFileHeader, printIssueRow, printEvaluationSummaries, type EvaluationSummary } from '../output/reporter'; -import { isJudgeResult } from '../prompts/schema'; import { handleUnknownError, MissingDependencyError } from '../errors/index'; import { processFindings } from '../findings'; import { createEvaluator } from '../evaluators/index'; @@ -28,7 +27,7 @@ import { calculateCost, TokenUsageStats } from '../providers/token-usage'; -import { calculateCheckScore } from '../scoring'; +import { calculateScore } from '../scoring'; import { countWords } from '../chunking/utils'; import { buildRuleId, normalizeRuleSource } from '../agent/rule-id'; import { @@ -170,12 +169,7 @@ function getViolationFilterResults< return { decisions, surfacedViolations }; } -/* - * Routes an evaluation result through the shared finding processor. - * Check results are verified, filtered, scored, and reported; judge/rubric - * results are rejected (Phase 3) because subjective scoring is not a - * future-facing review type. - */ +/** Routes an evaluation result through the shared finding processor. */ function routePromptResult( params: ProcessPromptResultParams ): ErrorTrackingResult { @@ -192,138 +186,101 @@ function routePromptResult( const meta = promptFile.meta; const promptId = (meta.id || "").toString(); - // Handle Check Result — routed through the shared finding processor - // (Phase 3, audit Findings #4 and #6). The processor verifies evidence, - // filters, deduplicates, scores, and resolves severity; the orchestrator - // only feeds the returned ReviewResult to the existing formatter sinks. - if (!isJudgeResult(result)) { - const reviewResult = processFindings({ - pack: promptFile.pack, - ruleId: promptId, - ruleSource: promptFile.fullPath, - candidateFindings: result.violations, - wordCount: result.word_count, - promptMeta: { - ...(meta.severity !== undefined ? { severity: meta.severity } : {}), - ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), - // Sanitize to the findings contract ({ id, name }): meta.criteria is - // PromptCriterionSpec[], which can carry legacy rubric weight/target. - ...(meta.criteria - ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } - : {}), - }, - targetContent: content, - }); + const reviewResult = processFindings({ + pack: promptFile.pack, + ruleId: promptId, + ruleSource: promptFile.fullPath, + candidateFindings: result.violations, + wordCount: result.word_count, + promptMeta: { + ...(meta.severity !== undefined ? { severity: meta.severity } : {}), + ...(meta.strictness !== undefined ? { strictness: meta.strictness } : {}), + ...(meta.criteria + ? { criteria: meta.criteria.map((c) => ({ id: c.id, name: c.name })) } + : {}), + }, + targetContent: content, + }); - // processFindings always returns exactly one score entry (processor contract). - const ruleScore = reviewResult.scores[0]!; - const severity = - ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; + const ruleScore = reviewResult.scores[0]!; + const severity = + ruleScore.severity === 'error' ? Severity.ERROR : Severity.WARNING; - // Report only verified findings through the existing line/json/rdjson/vale sink. - for (const finding of reviewResult.findings) { - reportIssue({ - file: relFile, - line: finding.line, - column: finding.column, - severity, - summary: finding.message, - ruleName: finding.ruleId, - outputFormat, - jsonFormatter, - ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), - ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), - ...(finding.fix !== undefined ? { fix: finding.fix } : {}), - match: finding.match, - }); - } + for (const finding of reviewResult.findings) { + reportIssue({ + file: relFile, + line: finding.line, + column: finding.column, + severity, + summary: finding.message, + ruleName: finding.ruleId, + outputFormat, + jsonFormatter, + ...(finding.analysis !== undefined ? { analysis: finding.analysis } : {}), + ...(finding.suggestion !== undefined ? { suggestion: finding.suggestion } : {}), + ...(finding.fix !== undefined ? { fix: finding.fix } : {}), + match: finding.match, + }); + } - // Diagnostics (e.g. finding-evidence-not-locatable) surface in verbose mode, - // consistent with prior operational reporting. They are warn-level and do not - // flag the run as operationally failed (audit Finding #6). - if (verbose) { - for (const diagnostic of reviewResult.diagnostics) { - console.warn(`[vectorlint] ${diagnostic.message}`); - } + if (verbose) { + for (const diagnostic of reviewResult.diagnostics) { + console.warn(`[vectorlint] ${diagnostic.message}`); } + } - // Verified finding count drives the counts (audit Finding #6). - const findingCount = reviewResult.findings.length; - const totalErrors = severity === Severity.ERROR ? findingCount : 0; - const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; + const findingCount = reviewResult.findings.length; + const totalErrors = severity === Severity.ERROR ? findingCount : 0; + const totalWarnings = severity === Severity.ERROR ? 0 : findingCount; - const scoreEntry: EvaluationSummary = { - id: ruleScore.ruleId, - scoreText: ruleScore.scoreText, - score: ruleScore.score, - }; + const scoreEntry: EvaluationSummary = { + id: ruleScore.ruleId, + scoreText: ruleScore.scoreText, + score: ruleScore.score, + }; - if (debugJson) { - const { decisions, surfacedViolations } = getViolationFilterResults( - result.violations - ); - const runId = randomUUID(); - const model = getModelInfoFromEnv(); - - try { - const filePath = writeDebugRunArtifact(process.cwd(), runId, { - file: relFile, - ...(Object.keys(model).length > 0 ? { model } : {}), - ...(model.tag !== undefined ? { subdir: model.tag } : {}), - prompt: { - pack: promptFile.pack, - id: promptId, - filename: promptFile.filename, - evaluation_type: "check", - }, - raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, - filter_decisions: decisions.map((d, i) => ({ - index: i, - surface: d.surface, - reasons: d.reasons, - })), - surfaced_violations: surfacedViolations, - }); - console.warn(`[vectorlint] Debug JSON written: ${filePath}`); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.warn(`[vectorlint] Debug JSON write failed: ${message}`); - } - } + if (debugJson) { + const { decisions, surfacedViolations } = getViolationFilterResults( + result.violations + ); + const runId = randomUUID(); + const model = getModelInfoFromEnv(); - return { - errors: totalErrors, - warnings: totalWarnings, - hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, - hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, - scoreEntries: [scoreEntry], - }; + try { + const filePath = writeDebugRunArtifact(process.cwd(), runId, { + file: relFile, + ...(Object.keys(model).length > 0 ? { model } : {}), + ...(model.tag !== undefined ? { subdir: model.tag } : {}), + prompt: { + pack: promptFile.pack, + id: promptId, + filename: promptFile.filename, + }, + raw_model_output: (result as { raw_model_output?: unknown }).raw_model_output ?? null, + filter_decisions: decisions.map((d, i) => ({ + index: i, + surface: d.surface, + reasons: d.reasons, + })), + surfaced_violations: surfacedViolations, + }); + console.warn(`[vectorlint] Debug JSON written: ${filePath}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[vectorlint] Debug JSON write failed: ${message}`); + } } - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // prompt-meta boundary rejects `type: judge`, so a JudgeResult here means a - // caller bypassed that boundary. Refuse it explicitly rather than - // misprojecting it as a check result. No adapter is added for judge metadata. - if (verbose) { - console.warn( - '[vectorlint] Judge/rubric review results are no longer supported; skipping prompt.' - ); - } return { - errors: 0, - warnings: 0, - hadOperationalErrors: true, - hadSeverityErrors: false, - scoreEntries: [], + errors: totalErrors, + warnings: totalWarnings, + hadOperationalErrors: reviewResult.hadOperationalErrors ?? false, + hadSeverityErrors: severity === Severity.ERROR && totalErrors > 0, + scoreEntries: [scoreEntry], }; } -/* - * Runs a single prompt evaluation. - * BaseEvaluator auto-detects mode from criteria presence: - * - criteria defined → scored mode - * - no criteria → basic mode - */ +/** Runs a single prompt evaluation. */ async function runPromptEvaluation( params: RunPromptEvaluationParams ): Promise { @@ -335,8 +292,7 @@ async function runPromptEvaluation( const evaluatorType = String(meta.evaluator || Type.BASE); const baseEvaluatorType = String(Type.BASE); - // Specialized evaluators (e.g., technical-accuracy) require criteria - // BaseEvaluator handles both modes: scored (with criteria) and basic (without) + // Specialized evaluators (e.g., technical-accuracy) require criteria. if (evaluatorType !== baseEvaluatorType) { if ( !meta || @@ -639,7 +595,7 @@ async function buildAgentRuleScores( analysis: 'Agent finding', })); - const scored = calculateCheckScore( + const scored = calculateScore( syntheticViolations, Math.max(1, totalWords), { @@ -813,9 +769,8 @@ export async function evaluateFiles( if (mode === AGENT_REVIEW_MODE) { options.logger?.warn( - '--mode agent is an unreleased internal path and now falls back to standard mode.', + '--mode agent falls back to standard evaluation.', ); - // Fall through to standard evaluation; do not call evaluateFilesInAgentMode. } for (const file of targets) { diff --git a/src/config/constants.ts b/src/config/constants.ts index ad9457ff..5a5dfa7d 100644 --- a/src/config/constants.ts +++ b/src/config/constants.ts @@ -20,7 +20,7 @@ export const ALLOWED_EXTS = new Set(EXTENSIONS.map(ext => `.${ext}`)); export const DEFAULT_CONCURRENCY = 4; export const DEFAULT_SCAN_PATTERN = `**/*.{${EXTENSIONS.join(',')}}`; -export const CLI_DESCRIPTION = `VectorLint is a command-line tool that evaluates and scores content using LLMs. It uses LLM-as-a-Judge to catch terminology, technical accuracy, and style issues that require contextual understanding. +export const CLI_DESCRIPTION = `VectorLint is a command-line tool that uses LLMs to find and score terminology, technical accuracy, and style issues that require contextual understanding. To get started, run 'vectorlint init' to create your configuration files. diff --git a/src/debug/run-artifact.ts b/src/debug/run-artifact.ts index cd6d496a..5f31d18d 100644 --- a/src/debug/run-artifact.ts +++ b/src/debug/run-artifact.ts @@ -14,7 +14,6 @@ export type DebugRunArtifact = { pack?: string; id?: string; filename?: string; - evaluation_type?: string; }; raw_model_output: unknown; filter_decisions: unknown; diff --git a/src/evaluators/accuracy-evaluator.ts b/src/evaluators/accuracy-evaluator.ts index 9a27b9fd..2634dffe 100644 --- a/src/evaluators/accuracy-evaluator.ts +++ b/src/evaluators/accuracy-evaluator.ts @@ -3,12 +3,12 @@ import { registerEvaluator } from "./evaluator-registry"; import type { LLMProvider } from "../providers/llm-provider"; import type { SearchProvider } from "../providers/search-provider"; import type { PromptFile } from "../schemas/prompt-schemas"; -import type { PromptEvaluationResult, RawCheckResult } from "../prompts/schema"; +import type { PromptEvaluationResult } from "../prompts/schema"; import type { TokenUsage } from "../providers/token-usage"; import { renderTemplate } from "../prompts/template-renderer"; import { getPrompt } from "./prompt-loader"; import { z } from "zod"; -import { Type, EvaluationType, type Severity } from "./types"; +import { Type, type Severity } from "./types"; import { MissingDependencyError } from "../errors/index"; import { countWords } from "../chunking"; @@ -64,8 +64,7 @@ export class TechnicalAccuracyEvaluator extends BaseEvaluator { // Use the scoring module to calculate result if (claims.length === 0) { const wordCount = countWords(content) || 1; - const raw: RawCheckResult = { - type: EvaluationType.CHECK, + const raw: PromptEvaluationResult = { violations: [], word_count: wordCount, ...(claimUsage && { usage: claimUsage }), @@ -87,7 +86,6 @@ export class TechnicalAccuracyEvaluator extends BaseEvaluator { const renderedPrompt = renderTemplate(this.getPromptBody(), templateVars); // Step 5: Create enriched prompt with rendered body - // We do NOT override the type here; we respect the prompt's configuration const enrichedPrompt: PromptFile = { ...this.prompt, body: renderedPrompt, diff --git a/src/evaluators/base-evaluator.ts b/src/evaluators/base-evaluator.ts index 4b84c391..2b71ae5f 100644 --- a/src/evaluators/base-evaluator.ts +++ b/src/evaluators/base-evaluator.ts @@ -4,43 +4,25 @@ import type { EvalContext } from "../providers/request-builder"; import type { PromptFile } from "../schemas/prompt-schemas"; import type { TokenUsage } from "../providers/token-usage"; import { - buildJudgeLLMSchema, - buildCheckLLMSchema, - type JudgeLLMResult, - type CheckLLMResult, - type JudgeResult, - type RawCheckResult, + buildEvaluationLLMSchema, + type EvaluationLLMResult, type PromptEvaluationResult, } from "../prompts/schema"; import { registerEvaluator } from "./evaluator-registry"; import type { Evaluator } from "./evaluator"; -import { Type, Severity, EvaluationType } from "./types"; +import { Type, Severity } from "./types"; import { mergeViolations, RecursiveChunker, countWords, type Chunk, } from "../chunking"; -import { - calculateJudgeScore, - averageJudgeScores, -} from "../scoring"; import { prependLineNumbers } from "../output/line-numbering"; const CHUNKING_THRESHOLD = 600; // Word count threshold for enabling chunking const MAX_CHUNK_SIZE = 500; // Maximum words per chunk -/* - * Core LLM-based evaluator that handles Judge and Check evaluation modes. - * Mode is determined by prompt frontmatter 'type' field: - * - 'judge': Weighted average of 1-4 scores per criterion, normalized to 1-10. - * - 'check': Density-based scoring (errors per 100 words). - * - * Content is automatically chunked for documents >600 words to improve accuracy. - * - * Subclasses can override protected methods to customize evaluation behavior - * while reusing the core evaluation logic. - */ +/** Evaluates rule violations, chunking large documents when needed. */ export class BaseEvaluator implements Evaluator { constructor( protected llmProvider: LLMProvider, @@ -49,28 +31,9 @@ export class BaseEvaluator implements Evaluator { ) { } async evaluate(file: string, content: string): Promise { - const type = this.getEvaluationType(); const ext = path.extname(file); const context: EvalContext = ext ? { fileType: ext } : {}; - - if (type === EvaluationType.JUDGE) { - return this.runJudgeEvaluation(content, context); - } else { - return this.runCheckEvaluation(content, context); - } - } - - /* - * Determines the evaluation type. - * Defaults to 'check' if not specified, for backward compatibility. - */ - protected getEvaluationType(): - | typeof EvaluationType.JUDGE - | typeof EvaluationType.CHECK { - // After Zod transform, type will be 'judge' or 'check' (or undefined) - return this.prompt.meta.type === "judge" - ? EvaluationType.JUDGE - : EvaluationType.CHECK; + return this.runEvaluation(content, context); } protected chunkContent(content: string): Chunk[] { @@ -110,94 +73,11 @@ export class BaseEvaluator implements Evaluator { ); } - /* - * Runs judge evaluation: - * 1. Prepend line numbers for accurate LLM line reporting. - * 2. Chunk content if needed. - * 3. LLM scores each criterion 1-4 for each chunk. - * 4. Average scores across chunks (weighted by chunk size). - */ - protected async runJudgeEvaluation( - content: string, - context?: EvalContext - ): Promise { - const schema = buildJudgeLLMSchema(); - - // Prepend line numbers for deterministic line reporting - const numberedContent = prependLineNumbers(content); - const chunks = this.chunkContent(numberedContent); - const usages: (TokenUsage | undefined)[] = []; - - // Single chunk - run directly - if (chunks.length === 1) { - const { data: llmResult, usage } = - await this.llmProvider.runPromptStructured( - numberedContent, - this.prompt.body, - schema, - context - ); - - const result = calculateJudgeScore(llmResult.criteria, { - promptCriteria: this.prompt.meta.criteria, - }); - - return { - ...result, - raw_model_output: llmResult, - ...(usage && { usage }), - }; - } - - // Multiple chunks - evaluate each and average - const chunkResults: JudgeResult[] = []; - const chunkWordCounts: number[] = []; - const rawChunkOutputs: JudgeLLMResult[] = []; - - for (const chunk of chunks) { - const { data: llmResult, usage } = - await this.llmProvider.runPromptStructured( - chunk.content, - this.prompt.body, - schema, - context - ); - - usages.push(usage); - rawChunkOutputs.push(llmResult); - - const result = calculateJudgeScore(llmResult.criteria, { - promptCriteria: this.prompt.meta.criteria, - }); - - chunkResults.push(result); - chunkWordCounts.push(countWords(chunk.content)); - } - - // Average scores across chunks - const result = averageJudgeScores(chunkResults, chunkWordCounts); - const aggregatedUsage = this.aggregateUsage(usages); - - return { - ...result, - raw_model_output: rawChunkOutputs, - ...(aggregatedUsage && { usage: aggregatedUsage }), - }; - } - - /* - * Runs check evaluation: - * 1. Prepend line numbers for accurate LLM line reporting. - * 2. Chunk content if needed. - * 3. LLM lists violations for each chunk. - * 4. Merge all violations across chunks. - * 5. Calculate score once from total violations. - */ - protected async runCheckEvaluation( + protected async runEvaluation( content: string, context?: EvalContext - ): Promise { - const schema = buildCheckLLMSchema(); + ): Promise { + const schema = buildEvaluationLLMSchema(); // Prepend line numbers for deterministic line reporting const numberedContent = prependLineNumbers(content); @@ -205,14 +85,14 @@ export class BaseEvaluator implements Evaluator { const totalWordCount = countWords(content) || 1; // Collect all violations from all chunks - const allChunkViolations: CheckLLMResult["violations"][] = []; - const rawChunkOutputs: CheckLLMResult[] = []; + const allChunkViolations: EvaluationLLMResult["violations"][] = []; + const rawChunkOutputs: EvaluationLLMResult[] = []; const chunkReasonings: string[] = []; const usages: (TokenUsage | undefined)[] = []; for (const chunk of chunks) { const { data: llmResult, usage } = - await this.llmProvider.runPromptStructured( + await this.llmProvider.runPromptStructured( chunk.content, this.prompt.body, schema, @@ -231,7 +111,6 @@ export class BaseEvaluator implements Evaluator { const reasoning = chunkReasonings.join(" ").trim() || undefined; return { - type: EvaluationType.CHECK, violations: mergedViolations, word_count: totalWordCount, ...(reasoning && { reasoning }), @@ -241,8 +120,6 @@ export class BaseEvaluator implements Evaluator { } } -// Register as default evaluator for base type -// Note: EvaluatorFactory signature is (llmProvider, prompt, searchProvider?, defaultSeverity?) registerEvaluator( Type.BASE, (llmProvider, prompt, _searchProvider, defaultSeverity) => { diff --git a/src/evaluators/types.ts b/src/evaluators/types.ts index c78797fc..b17a1686 100644 --- a/src/evaluators/types.ts +++ b/src/evaluators/types.ts @@ -10,8 +10,3 @@ export enum Severity { ERROR = 'error', WARNING = 'warning', } - -export enum EvaluationType { - JUDGE = 'judge', - CHECK = 'check', -} diff --git a/src/findings/finding-evidence-verifier.ts b/src/findings/finding-evidence-verifier.ts index e2d6df9e..bdefc866 100644 --- a/src/findings/finding-evidence-verifier.ts +++ b/src/findings/finding-evidence-verifier.ts @@ -8,7 +8,7 @@ export interface FindingEvidenceInput { quoted_text: string; context_before?: string; context_after?: string; - /** Optional 1-based line hint from the model; never used as a fallback. */ + /** Optional 1-based line hint from the model. */ line?: number; } @@ -36,17 +36,7 @@ export interface FindingEvidenceVerification { diagnostic?: FindingEvidenceDiagnostic; } -/** - * The single source of truth for finding evidence verification - * (audit Finding #6). - * - * Wraps the existing `locateQuotedText` fuzzy matcher; it never reimplements - * matching. On a successful anchor it returns a verified finding with the - * located line/column/match. When the quoted text cannot be anchored it - * returns a `finding-evidence-not-locatable` warn diagnostic and NO finding — - * it never falls back to a model-provided line number the way the unreleased - * internal agent implementation did. - */ +/** Verifies and locates a candidate finding's quoted evidence. */ export function verifyFindingEvidence( content: string, findingEvidence: FindingEvidenceInput, diff --git a/src/findings/index.ts b/src/findings/index.ts index c3a9a666..a22e0341 100644 --- a/src/findings/index.ts +++ b/src/findings/index.ts @@ -1,14 +1,4 @@ -/** - * Shared finding-processing layer (Phase 3). - * - * Transforms raw candidate violation findings into a formatter-ready - * `ReviewResult` through one pipeline: evidence verification, filtering, - * count/density scoring, severity resolution, and diagnostics. - * - * This module is independent of model call (`single` | `agent`) and contains - * no `judge`, `evaluator`, rubric, or autonomous-loop output - * adapter code. - */ +/** Shared candidate-to-finding processing pipeline. */ export { FINDING_EVIDENCE_NOT_LOCATABLE, verifyFindingEvidence, @@ -25,7 +15,7 @@ export { type SeverityInput, } from './severity'; -export { scoreCheck, type ScoreCheckOptions, type ScoredCheck } from './scorer'; +export { scoreFindings, type ScoreOptions, type ScoredFindings } from './scorer'; export { processFindings, diff --git a/src/findings/processor.ts b/src/findings/processor.ts index 45a1c7ae..d4f9c72f 100644 --- a/src/findings/processor.ts +++ b/src/findings/processor.ts @@ -10,7 +10,7 @@ import { FINDING_EVIDENCE_NOT_LOCATABLE, type VerifiedFindingCoords, } from './finding-evidence-verifier'; -import { scoreCheck } from './scorer'; +import { scoreFindings } from './scorer'; import { buildRuleId, resolveCriterionId } from './severity'; import type { FindingProcessingInput, RawViolation } from './types'; @@ -19,24 +19,7 @@ interface VerifiedEntry { coords: VerifiedFindingCoords; } -/** - * The single finding-processing pipeline for objective Via Negativa violation - * checks (Phase 3). Transforms raw candidate findings into a formatter-ready - * {@link ReviewResult}. - * - * Order (audit lines 167-174): - * 1. filter candidates by the evidence gate (`computeFilterDecision`); - * 2. verify each surfaced finding's evidence (anchored quote or diagnostic); - * 3. deduplicate verified findings by `(quoted_text, line)`; - * 4. score by verified finding count/density; - * 5. resolve severity from the density score; - * 6. assemble findings, score, and diagnostics. - * - * The verified finding count — not the raw candidate count — drives the score - * (audit Finding #6). Unanchored evidence becomes a - * `finding-evidence-not-locatable` warn diagnostic and emits no finding. - * `hadOperationalErrors` is `true` only when a diagnostic is `error` level. - */ +/** Transforms candidate violations into a formatter-ready {@link ReviewResult}. */ export function processFindings(input: FindingProcessingInput): ReviewResult { const surfaced = filterCandidates(input.candidateFindings); const { verified, evidenceDiagnostics } = verifyAndDedupe( @@ -44,7 +27,7 @@ export function processFindings(input: FindingProcessingInput): ReviewResult { input.targetContent, ); - const scored = scoreCheck({ + const scored = scoreFindings({ verifiedViolations: verified.map((entry) => entry.raw), wordCount: input.wordCount, ...(input.promptMeta.strictness !== undefined @@ -97,11 +80,7 @@ export function processFindings(input: FindingProcessingInput): ReviewResult { return { findings, scores, diagnostics, hadOperationalErrors }; } -/** - * Drops candidate findings that fail the evidence gate. Filtered candidates - * contribute neither findings nor diagnostics, matching the current standard - * orchestrator behavior. - */ +/** Drops candidate findings that fail the evidence gate. */ function filterCandidates(candidates: readonly RawViolation[]): RawViolation[] { const surfaced: RawViolation[] = []; for (const candidate of candidates) { diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts index b5fb93fa..29f6be15 100644 --- a/src/findings/scorer.ts +++ b/src/findings/scorer.ts @@ -1,45 +1,29 @@ -import { calculateCheckScore, type CheckScoringOptions } from '../scoring'; -import type { CheckResult } from '../prompts/schema'; +import { calculateScore, type ScoringOptions } from '../scoring'; +import type { ScoredEvaluation } from '../prompts/schema'; import { resolveSeverity } from './severity'; import type { RawViolation, RuleSeverity } from './types'; -/** Optional scoring knobs mirroring {@link CheckScoringOptions}. */ -export interface ScoreCheckOptions { - strictness?: CheckScoringOptions['strictness']; +export interface ScoreOptions { + strictness?: ScoringOptions['strictness']; promptSeverity?: RuleSeverity; } -/** - * The count/density score result for a rule's verified findings. Mirrors the - * fields the processor needs to assemble a `ReviewScore`. The numeric values - * come straight from {@link calculateCheckScore}; this layer never - * reimplements the scoring math (audit Finding #4). - */ -export interface ScoredCheck { +export interface ScoredFindings { score: number; scoreText: string; severity: RuleSeverity; - /** Count of verified findings that drove the score. */ findingCount: number; - /** The raw check result, used to resolve severity. */ - scored: CheckResult; + scored: ScoredEvaluation; } -/** - * Scores a rule's verified findings using the existing check density formula. - * - * The score is driven by the **verified** finding count, not the raw candidate - * count (audit Finding #6): callers must pass only findings whose evidence has - * been anchored. This is a thin adapter over {@link calculateCheckScore}; it - * only normalizes the result into the shared score shape. - */ -export function scoreCheck(params: { +/** Scores verified findings using violation density. */ +export function scoreFindings(params: { verifiedViolations: RawViolation[]; wordCount: number; - strictness?: CheckScoringOptions['strictness']; + strictness?: ScoringOptions['strictness']; promptSeverity?: RuleSeverity; -}): ScoredCheck { - const scored = calculateCheckScore( +}): ScoredFindings { + const scored = calculateScore( params.verifiedViolations, params.wordCount, { diff --git a/src/findings/severity.ts b/src/findings/severity.ts index cca534f0..6bb448bc 100644 --- a/src/findings/severity.ts +++ b/src/findings/severity.ts @@ -1,27 +1,17 @@ -import type { CheckResult } from '../prompts/schema'; +import type { ScoredEvaluation } from '../prompts/schema'; import type { FindingsCriterion, RuleSeverity } from './types'; /** Input to {@link resolveSeverity}. */ export interface SeverityInput { - scored: CheckResult; + scored: ScoredEvaluation; } -/** - * The one severity resolution path for objective violation checks - * (audit Finding #4). Severity is derived from the existing count/density - * score result (`CheckResult.severity`); there is no agent-specific or - * mode-specific severity stamping here. - */ +/** Resolves severity from a scored evaluation. */ export function resolveSeverity(input: SeverityInput): RuleSeverity { return input.scored.severity; } -/** - * Builds the hierarchical output rule id matching the current standard - * orchestrator naming (`buildRuleName`): `Pack.Rule` or, when a violation is - * attributed to a criterion, `Pack.Rule.Criterion`. This is a faithful port of - * that behavior; it does not import from `src/agent/`. - */ +/** Builds a `Pack.Rule` or `Pack.Rule.Criterion` output id. */ export function buildRuleId( pack: string, ruleId: string, @@ -38,7 +28,7 @@ export function buildRuleId( * Resolves a criterion id from the rule's declared criteria by matching a * `criterionName` a violation carries to a declared criterion's `name`. * Returns `undefined` when the violation is not attributed to a declared - * criterion (the common case for objective check violations). + * criterion. */ export function resolveCriterionId( criteria: readonly FindingsCriterion[] | undefined, diff --git a/src/findings/types.ts b/src/findings/types.ts index 9a6360c6..9c8c0b85 100644 --- a/src/findings/types.ts +++ b/src/findings/types.ts @@ -1,29 +1,10 @@ import { z } from 'zod'; import { Severity } from '../evaluators/types'; -/** - * Input contract for shared finding processing (Phase 3, audit Finding #6). - * - * This module is the boundary between "what a model returned" (raw candidate - * violations) and "what the formatters receive" (`ReviewResult`). It is - * intentionally independent of model call (`single` | `agent`): there is no - * `modelCall`, `mode`, `agent`, `evaluator`, `judge`, or rubric field here. - * - * Following the `src/review/` pattern, the domain types below are explicit - * interfaces (so their optional properties stay assignable under - * `exactOptionalPropertyTypes` to `CheckItem`/`GateViolationLike`), and the - * Zod schemas mirror them for boundary validation. The schemas are strict so - * legacy `evaluator`/`judge`/`modelCall`/`mode`/`agent`/rubric payloads are - * rejected. - */ - -/** - * The two severity levels objective violation checks can resolve to. Mirrors - * `ReviewSeverity`; values are identical (`'error'` | `'warning'`). - */ +/** Severity assigned to a rule and its findings. */ export type RuleSeverity = typeof Severity.ERROR | typeof Severity.WARNING; -/** Optional strictness knob for check density scoring. */ +/** Optional strictness knob for density scoring. */ export type Strictness = number | 'lenient' | 'standard' | 'strict'; /** @@ -62,33 +43,20 @@ export interface RawViolation { checks?: GateChecks; } -/** - * Criterion metadata used ONLY to name output rule ids - * (`Pack.Rule.Criterion`). Weight and target are judge/rubric concerns and are - * deliberately absent so the finding-processing contract cannot leak them. - */ +/** Criterion metadata used to name `Pack.Rule.Criterion` output ids. */ export interface FindingsCriterion { id: string; name: string; } -/** - * The narrow slice of prompt metadata that finding processing needs: severity - * for scoring, strictness for density, and criteria for rule-id naming. It - * excludes `evaluator`, `type`, `evaluateAs`, `target`, and other legacy - * fields that do not affect objective violation-check processing. - */ +/** Prompt metadata used by finding processing. */ export interface PromptMetaForFindings { severity?: RuleSeverity; strictness?: Strictness; criteria?: FindingsCriterion[]; } -/** - * The full finding-processing input. There is deliberately no `outputFormat`: - * the processor emits a format-agnostic `ReviewResult`; the caller routes it - * to a formatter. - */ +/** Format-independent input for finding processing. */ export interface FindingProcessingInput { pack: string; ruleId: string; @@ -99,8 +67,6 @@ export interface FindingProcessingInput { targetContent: string; } -// --- Boundary validation schemas (mirror the interfaces above) --------------- - export const GATE_CHECKS_SCHEMA = z .object({ rule_supports_claim: z.boolean(), diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index eeddac6e..0d589d14 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -4,8 +4,6 @@ export interface ScoreComponent { criterion?: string; rawScore: number; maxScore: number; - weightedScore: number; - weightedMaxScore: number; normalizedScore: number; normalizedMaxScore: number; } diff --git a/src/prompts/prompt-validator.ts b/src/prompts/prompt-validator.ts index 3c584469..55897553 100644 --- a/src/prompts/prompt-validator.ts +++ b/src/prompts/prompt-validator.ts @@ -39,26 +39,6 @@ export function validatePrompt(p: PromptFile): Validation[] { if (!c.id && !c.name) { out.push({ file: p.filename, level: Severity.ERROR, message: 'Criterion missing id/name' }); } - const w = Number(c.weight); - if (!Number.isFinite(w) || w <= 0) { - out.push({ file: p.filename, level: Severity.ERROR, message: `Invalid weight for ${c.name || c.id}` }); - } - if (c.target) { - if (!validateFlags(c.target.flags)) { - out.push({ file: p.filename, level: Severity.ERROR, message: `Invalid regex flags in target for ${c.name || c.id}` }); - } - if (c.target.group !== undefined && (!Number.isInteger(c.target.group) || c.target.group < 0)) { - out.push({ file: p.filename, level: Severity.ERROR, message: `Invalid target.group for ${c.name || c.id}` }); - } - if (c.target.regex) { - try { new RegExp(c.target.regex, c.target.flags || ''); } catch { - out.push({ file: p.filename, level: Severity.ERROR, message: `Invalid target.regex for ${c.name || c.id}` }); - } - } - if (c.target.suggestion && typeof c.target.suggestion !== 'string') { - out.push({ file: p.filename, level: Severity.ERROR, message: `Invalid target.suggestion for ${c.name || c.id}` }); - } - } } const dups = uniqueIds(meta.criteria); diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index bd9fd9ee..a02972ff 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -1,4 +1,4 @@ -import { EvaluationType, Severity } from "../evaluators/types"; +import { Severity } from "../evaluators/types"; import type { TokenUsage } from "../providers/token-usage"; export type GateChecks = { @@ -19,139 +19,9 @@ export type GateCheckNotes = { fix_preserves_meaning: string; }; -export function buildJudgeLLMSchema() { +export function buildEvaluationLLMSchema() { return { - name: "vectorlint_judge_result", - strict: true, - schema: { - type: "object", - additionalProperties: false, - properties: { - criteria: { - type: "array", - items: { - type: "object", - additionalProperties: false, - properties: { - reasoning: { - type: "string", - description: - "Thorough step-by-step logic for the evaluation of this specific criterion.", - }, - name: { type: "string" }, - summary: { type: "string" }, - score: { type: "number", enum: [1, 2, 3, 4] }, - violations: { - type: "array", - items: { - type: "object", - additionalProperties: false, - properties: { - line: { type: "number" }, - quoted_text: { type: "string" }, - context_before: { type: "string" }, - context_after: { type: "string" }, - description: { type: "string" }, - analysis: { - type: "string", - description: - "A concise 1-2 sentence explanation of the specific issue.", - }, - message: { - type: "string", - description: - "Under 15 words. State the issue directly to the user. No rule references.", - }, - suggestion: { - type: "string", - description: "Suggest a fix in 15 words or less.", - }, - fix: { type: "string" }, - rule_quote: { type: "string" }, - checks: { - type: "object", - additionalProperties: false, - properties: { - rule_supports_claim: { type: "boolean" }, - evidence_exact: { type: "boolean" }, - context_supports_violation: { type: "boolean" }, - plausible_non_violation: { - type: "boolean", - description: - "true if a reasonable reading of the context could explain the pattern as intentional or acceptable — even if you believe a violation exists. When in doubt, mark true.", - }, - fix_is_drop_in: { type: "boolean" }, - fix_preserves_meaning: { type: "boolean" }, - }, - required: [ - "rule_supports_claim", - "evidence_exact", - "context_supports_violation", - "plausible_non_violation", - "fix_is_drop_in", - "fix_preserves_meaning", - ], - }, - check_notes: { - type: "object", - additionalProperties: false, - properties: { - rule_supports_claim: { type: "string" }, - evidence_exact: { type: "string" }, - context_supports_violation: { type: "string" }, - plausible_non_violation: { - type: "string", - description: - "Describe a concrete scenario or context in which this same pattern would NOT be a violation. Do not simply restate that no non-violation exists — always articulate what conditions would make this acceptable.", - }, - fix_is_drop_in: { type: "string" }, - fix_preserves_meaning: { type: "string" }, - }, - required: [ - "rule_supports_claim", - "evidence_exact", - "context_supports_violation", - "plausible_non_violation", - "fix_is_drop_in", - "fix_preserves_meaning", - ], - }, - confidence: { - type: "number", - description: - "A score from 0.0 to 1.0 reflecting how certain you are this is a genuine violation. Findings where plausible_non_violation is true, context_supports_violation is weak, or the rule match is approximate should score below 0.7. Reserve scores above 0.9 for unambiguous violations with no credible alternative reading.", - }, - }, - required: [ - "line", - "quoted_text", - "context_before", - "context_after", - "description", - "analysis", - "message", - "suggestion", - "fix", - "rule_quote", - "checks", - "check_notes", - "confidence", - ], - }, - }, - }, - required: ["reasoning", "name", "score", "summary", "violations"], - }, - }, - }, - required: ["criteria"], - }, - } as const; -} - -export function buildCheckLLMSchema() { - return { - name: "vectorlint_check_result", + name: "vectorlint_evaluation_result", strict: true, schema: { type: "object", @@ -262,31 +132,7 @@ export function buildCheckLLMSchema() { } as const; } -export type JudgeLLMResult = { - criteria: Array<{ - name: string; - score: 1 | 2 | 3 | 4; - summary: string; - reasoning: string; - violations: Array<{ - line: number; - quoted_text: string; - context_before: string; - context_after: string; - description: string; - analysis: string; - message: string; - suggestion: string; - fix: string; - rule_quote: string; - checks: GateChecks; - check_notes: GateCheckNotes; - confidence: number; - }>; - }>; -}; - -export type CheckLLMResult = { +export type EvaluationLLMResult = { reasoning: string; violations: Array<{ line: number; @@ -305,38 +151,7 @@ export type CheckLLMResult = { }>; }; -export type JudgeResult = { - type: typeof EvaluationType.JUDGE; - final_score: number; // 1-10 - criteria: Array<{ - name: string; - weight: number; - score: 1 | 2 | 3 | 4; - normalized_score: number; - weighted_points: number; - summary: string; - reasoning: string; - violations: Array<{ - line: number; - quoted_text: string; - context_before: string; - context_after: string; - description: string; - analysis: string; - message: string; - suggestion: string; - fix: string; - rule_quote: string; - checks: GateChecks; - check_notes: GateCheckNotes; - confidence: number; - }>; - }>; - usage?: TokenUsage; - raw_model_output?: unknown; -}; - -export type CheckItem = { +export type EvaluationItem = { line?: number; description?: string; analysis: string; @@ -352,12 +167,11 @@ export type CheckItem = { confidence?: number; }; -export type CheckResult = { - type: typeof EvaluationType.CHECK; +export type ScoredEvaluation = { final_score: number; // 1-10 percentage: number; violation_count: number; - items: Array; + items: Array; severity: typeof Severity.WARNING | typeof Severity.ERROR; message: string; reasoning?: string; @@ -381,27 +195,11 @@ export type CheckResult = { raw_model_output?: unknown; }; -export type RawCheckResult = { - type: typeof EvaluationType.CHECK; - violations: CheckResult["violations"]; +export type PromptEvaluationResult = { + violations: ScoredEvaluation["violations"]; word_count: number; reasoning?: string; usage?: TokenUsage; raw_model_output?: unknown; }; -export type PromptEvaluationResult = JudgeResult | RawCheckResult; - -export function isJudgeResult( - result: PromptEvaluationResult -): result is JudgeResult { - return result.type === EvaluationType.JUDGE; -} - -export function isCheckResult( - result: PromptEvaluationResult -): result is RawCheckResult { - return result.type === EvaluationType.CHECK; -} - - diff --git a/src/prompts/target.ts b/src/prompts/target.ts index 597812e9..9224b616 100644 --- a/src/prompts/target.ts +++ b/src/prompts/target.ts @@ -5,22 +5,20 @@ import type { TargetSpec } from '../schemas/prompt-schemas'; export function checkTarget( content: string, - metaTarget?: TargetSpec, - criterionTarget?: TargetSpec + target?: TargetSpec, ): { missing: boolean; suggestion?: string | undefined } { - const tgt = criterionTarget ?? metaTarget; - if (!tgt || !tgt.regex) return { missing: false }; + if (!target || !target.regex) return { missing: false }; let match: RegExpExecArray | null = null; try { - const flags = tgt.flags || 'mu'; - const re = new RegExp(tgt.regex, flags); + const flags = target.flags || 'mu'; + const re = new RegExp(target.regex, flags); match = re.exec(content); } catch { // invalid regex: treat as missing only if required - return { missing: !!tgt.required, suggestion: tgt.suggestion }; + return { missing: !!target.required, suggestion: target.suggestion }; } if (!match) { - return { missing: !!tgt.required, suggestion: tgt.suggestion }; + return { missing: !!target.required, suggestion: target.suggestion }; } return { missing: false }; } diff --git a/src/review/schemas.ts b/src/review/schemas.ts index 398c8803..930c6b9e 100644 --- a/src/review/schemas.ts +++ b/src/review/schemas.ts @@ -61,7 +61,6 @@ export const REVIEW_SCORE_COMPONENT_SCHEMA = z id: z.string().min(1), scoreText: z.string(), score: z.number(), - weight: z.number().optional(), }) .strict(); diff --git a/src/review/types.ts b/src/review/types.ts index 2df19e08..24381c2c 100644 --- a/src/review/types.ts +++ b/src/review/types.ts @@ -136,7 +136,6 @@ export interface ReviewScoreComponent { id: string; scoreText: string; score: number; - weight?: number; } export type ReviewDiagnosticLevel = 'info' | 'warn' | 'error'; diff --git a/src/schemas/prompt-schemas.ts b/src/schemas/prompt-schemas.ts index 2dadd85e..cb246335 100644 --- a/src/schemas/prompt-schemas.ts +++ b/src/schemas/prompt-schemas.ts @@ -16,56 +16,12 @@ export const TARGET_SPEC_SCHEMA = z export const PROMPT_CRITERION_SCHEMA = z.object({ id: z.string(), name: z.string(), - weight: z.number().positive().optional(), - target: TARGET_SPEC_SCHEMA.optional(), }); -// Prompt metadata schema for YAML frontmatter -/* - * Evaluator type selection: - * - 'base': unified evaluator (auto-detects scored vs basic mode from criteria) - * - 'technical-accuracy': specialized evaluator with claim extraction + search - * - * Evaluation type: - * - 'check': density-based scoring (errors per 100 words) - * - 'semi-objective': deprecated alias for 'check' - * - * 'judge' and its deprecated alias 'subjective' are no longer supported - * (Phase 3): subjective rubric scoring is not a future-facing review type. - * The enum still admits them so the Zod error is descriptive and the inferred - * type stays compatible with the legacy BaseEvaluator until Phase 4 deletes - * that path; a superRefine rejects them at this boundary so judge-typed prompts - * fail to load. - * - * Strictness factor for check scoring: - * - Determines penalty weight per 1% error density. - * - Default: 10 - */ +// Prompt metadata schema for YAML frontmatter. export const PROMPT_META_SCHEMA = z.object({ specVersion: z.union([z.string(), z.number()]).optional(), evaluator: z.enum(["base", "technical-accuracy"]).optional(), - type: z - .enum(["judge", "check", "subjective", "semi-objective"]) - .transform((val) => { - // Map deprecated values to their canonical forms. - if (val === "subjective") return "judge" as const; - if (val === "semi-objective") return "check" as const; - return val; - }) - .superRefine((val, ctx) => { - // Judge/rubric reviews are not a future-facing review type (Phase 3). - // superRefine (not refine) so the inferred union still includes 'judge' - // and stays compatible with the legacy BaseEvaluator until Phase 4 deletes - // that path; the value is rejected at this boundary regardless. - if (val === "judge") { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: - "judge evaluation is no longer supported; use check (Via Negativa) rules", - }); - } - }) - .optional(), id: z.string(), name: z.string(), severity: z.nativeEnum(Severity).optional(), @@ -74,7 +30,6 @@ export const PROMPT_META_SCHEMA = z.object({ .optional(), target: TARGET_SPEC_SCHEMA.optional(), criteria: z.array(PROMPT_CRITERION_SCHEMA).optional(), - // Determines how content is evaluated: 'chunk' (default) for chunked processing, 'document' for full document evaluateAs: z.enum(["document", "chunk"]).optional(), }); diff --git a/src/scoring/index.ts b/src/scoring/index.ts index 0ce54580..28402547 100644 --- a/src/scoring/index.ts +++ b/src/scoring/index.ts @@ -1,7 +1 @@ -export { - type CheckScoringOptions, - type JudgeScoringOptions, - calculateCheckScore, - calculateJudgeScore, - averageJudgeScores, -} from "./scorer"; +export { type ScoringOptions, calculateScore } from "./scorer"; diff --git a/src/scoring/scorer.ts b/src/scoring/scorer.ts index 24f0ced5..b0560b52 100644 --- a/src/scoring/scorer.ts +++ b/src/scoring/scorer.ts @@ -1,24 +1,12 @@ -import type { - CheckResult, - JudgeResult, - CheckItem, - JudgeLLMResult, -} from "../prompts/schema"; -import { EvaluationType, Severity } from "../evaluators/types"; +import type { EvaluationItem, ScoredEvaluation } from "../prompts/schema"; +import { Severity } from "../evaluators/types"; -export interface CheckScoringOptions { - // Strictness factor. Higher = more penalty per violation. +export interface ScoringOptions { strictness?: number | "lenient" | "strict" | "standard" | undefined; defaultSeverity?: typeof Severity.WARNING | typeof Severity.ERROR | undefined; promptSeverity?: typeof Severity.WARNING | typeof Severity.ERROR | undefined; } -export interface JudgeScoringOptions { - promptCriteria?: - | Array<{ name: string; weight?: number | undefined }> - | undefined; -} - function resolveStrictness( config: number | "lenient" | "strict" | "standard" | undefined ): number { @@ -37,33 +25,25 @@ function resolveStrictness( } /** - * Calculates check score based on violation density. + * Calculates a score from violation density. * * Formula: Score = (100 - (violations/wordCount * 100 * strictness)) / 10 */ -export function calculateCheckScore( - violations: CheckItem[], +export function calculateScore( + violations: EvaluationItem[], wordCount: number, - options: CheckScoringOptions = {} -): CheckResult { + options: ScoringOptions = {} +): ScoredEvaluation { const strictness = resolveStrictness(options.strictness); - const mappedViolations = violations.map((item) => ({ ...item, ...(item.description !== undefined ? { criterionName: item.description } : {}), })); - - // Density Calculation: Violations per 100 words const density = (mappedViolations.length / wordCount) * 100; - - // Score Calculation: 100 - (Density * Strictness), clamped 0-100 const rawScore = Math.max(0, Math.min(100, 100 - density * strictness)); const finalScore = rawScore / 10; - // Determine severity - let severity: typeof Severity.WARNING | typeof Severity.ERROR = - Severity.WARNING; - + let severity: typeof Severity.WARNING | typeof Severity.ERROR = Severity.WARNING; if (finalScore < 10) { if (options.promptSeverity !== undefined) { severity = options.promptSeverity; @@ -74,12 +54,10 @@ export function calculateCheckScore( const message = mappedViolations.length > 0 - ? `Found ${mappedViolations.length} issue${mappedViolations.length > 1 ? "s" : "" - }` + ? `Found ${mappedViolations.length} issue${mappedViolations.length > 1 ? "s" : ""}` : "No issues found"; return { - type: EvaluationType.CHECK, final_score: Number(finalScore.toFixed(1)), percentage: Number(rawScore.toFixed(1)), violation_count: mappedViolations.length, @@ -89,174 +67,3 @@ export function calculateCheckScore( severity, }; } - -/** - * Calculates judge score from criteria results. - * - * Each criterion score (1-4) is normalized to 1-10 scale, - * then weighted average is calculated. - */ -export function calculateJudgeScore( - criteria: JudgeLLMResult["criteria"], - options: JudgeScoringOptions = {} -): JudgeResult { - let totalWeightedScore = 0; - let totalWeight = 0; - - const criteriaWithCalculations = criteria.map((c) => { - // Find weight from prompt definition - const definedCriterion = options.promptCriteria?.find( - (dc) => dc.name === c.name - ); - const weight = definedCriterion?.weight || 1; - - // Normalize 1-4 score to 1-10 scale - const normalizedScore = 1 + ((c.score - 1) / 3) * 9; - const weightedPoints = normalizedScore * weight; - - totalWeightedScore += weightedPoints; - totalWeight += weight; - - return { - ...c, - weight, - normalized_score: Number(normalizedScore.toFixed(2)), - weighted_points: Number(weightedPoints.toFixed(2)), - }; - }); - - const finalScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 1; - - return { - type: EvaluationType.JUDGE, - final_score: Number(finalScore.toFixed(1)), - criteria: criteriaWithCalculations, - }; -} - -// Averages judge scores from multiple chunk evaluations. -export function averageJudgeScores( - results: JudgeResult[], - chunkWordCounts: number[] -): JudgeResult { - if (results.length === 0) { - return { - type: EvaluationType.JUDGE, - final_score: 0, - criteria: [], - }; - } - - // Warn if array lengths don't match (indicates a programming error) - if (results.length !== chunkWordCounts.length) { - console.warn( - `[vectorlint] Array length mismatch in averageJudgeScores: ` + - `${results.length} results vs ${chunkWordCounts.length} word counts` - ); - } - - const totalWords = chunkWordCounts.reduce((a, b) => a + b, 0); - - type JudgeViolation = JudgeResult["criteria"][number]["violations"][number]; - - // Aggregate criteria scores weighted by chunk size - const criteriaMap = new Map< - string, - { - totalScore: number; - totalWeight: number; - weight: number; - violations: JudgeViolation[]; - summaries: string[]; - reasonings: string[]; - } - >(); - - for (let i = 0; i < results.length; i++) { - const result = results[i]; - const chunkWeight = - totalWords > 0 - ? (chunkWordCounts[i] || 0) / totalWords - : 1 / results.length; - - for (const criterion of result?.criteria || []) { - if (!criteriaMap.has(criterion.name)) { - criteriaMap.set(criterion.name, { - totalScore: 0, - totalWeight: 0, - weight: criterion.weight || 1, - violations: [], - summaries: [], - reasonings: [], - }); - } - - const entry = criteriaMap.get(criterion.name)!; - entry.totalScore += criterion.score * chunkWeight; - entry.totalWeight += chunkWeight; - - // Collect raw violations (keeps all gate fields) - for (const v of criterion.violations || []) { - entry.violations.push(v); - } - - if (criterion.summary) { - entry.summaries.push(criterion.summary); - } - if (criterion.reasoning) { - entry.reasonings.push(criterion.reasoning); - } - } - } - - // Build aggregated criteria - const aggregatedCriteria: JudgeResult["criteria"] = []; - let totalWeightedScore = 0; - let totalWeight = 0; - - for (const [name, entry] of Array.from(criteriaMap.entries())) { - const avgScore = - entry.totalWeight > 0 ? entry.totalScore / entry.totalWeight : 0; - const roundedScore = Math.max(1, Math.min(4, Math.round(avgScore))) as - | 1 - | 2 - | 3 - | 4; - const normalizedScore = 1 + ((roundedScore - 1) / 3) * 9; - const weightedPoints = normalizedScore * entry.weight; - - totalWeightedScore += weightedPoints; - totalWeight += entry.weight; - - // Deduplicate violations using composite key - const seen = new Set(); - const uniqueViolations = entry.violations.filter((v) => { - const key = [ - v.quoted_text?.toLowerCase().trim() || "", - v.analysis?.toLowerCase().trim() || "", - ].join("|"); - if (seen.has(key)) return false; - seen.add(key); - return true; - }); - - aggregatedCriteria.push({ - name, - score: roundedScore, - weight: entry.weight, - normalized_score: Number(normalizedScore.toFixed(2)), - weighted_points: Number(weightedPoints.toFixed(2)), - summary: entry.summaries.join(" "), - reasoning: entry.reasonings.join(" "), - violations: uniqueViolations, - }); - } - - const finalScore = totalWeight > 0 ? totalWeightedScore / totalWeight : 0; - - return { - type: EvaluationType.JUDGE, - final_score: Number(finalScore.toFixed(1)), - criteria: aggregatedCriteria, - }; -} diff --git a/tests/agent/agent-executor.test.ts b/tests/agent/agent-executor.test.ts index 2f3d1ec5..e3a47ef2 100644 --- a/tests/agent/agent-executor.test.ts +++ b/tests/agent/agent-executor.test.ts @@ -19,7 +19,6 @@ function makePrompt(): PromptFile { meta: { id: 'Consistency', name: 'Consistency', - type: 'check', severity: Severity.WARNING, }, }; @@ -666,103 +665,6 @@ describe('agent executor', () => { expect(promptBodies[0]).toBe('Find inconsistent wording.'); }); - it('records judge-style violations as inline findings in agent mode', async () => { - const { runAgentExecutor } = await import('../../src/agent/executor'); - - const repo = createTempRepo(); - writeFileSync(path.join(repo, 'doc.md'), 'bad phrase\n', 'utf8'); - - const basePrompt = makePrompt(); - const judgePrompt: PromptFile = { - ...basePrompt, - body: 'Judge the document for clarity.', - meta: { - ...basePrompt.meta, - type: 'judge', - criteria: [{ id: 'Clarity', name: 'Clarity', weight: 1 }], - }, - }; - - const provider: LLMProvider = { - runPromptStructured() { - return Promise.resolve({ - data: { - criteria: [ - { - name: 'Clarity', - score: 2, - summary: 'Needs work', - reasoning: 'The wording is unclear.', - violations: [ - { - line: 1, - quoted_text: 'bad phrase', - context_before: '', - context_after: '', - description: 'Unclear wording', - analysis: 'This phrase is vague.', - message: 'Use clearer wording', - suggestion: 'Replace the vague phrase', - fix: 'better phrase', - rule_quote: 'Prefer precise language', - checks: { - rule_supports_claim: true, - evidence_exact: true, - context_supports_violation: true, - plausible_non_violation: false, - fix_is_drop_in: true, - fix_preserves_meaning: true, - }, - check_notes: { - rule_supports_claim: 'clear', - evidence_exact: 'exact', - context_supports_violation: 'yes', - plausible_non_violation: 'none', - fix_is_drop_in: 'yes', - fix_preserves_meaning: 'yes', - }, - confidence: 0.95, - }, - ], - }, - ], - }, - }); - }, - runAgentToolLoop: async (params: Record) => { - const tools = params.tools as Record Promise }>; - await tools.lint.execute({ - file: 'doc.md', - ruleSource: 'packs/default/consistency.md', - }); - await tools.finalize_review.execute({}); - return { usage: { inputTokens: 1, outputTokens: 1 } }; - }, - } as unknown as LLMProvider; - - const result = await runAgentExecutor({ - targets: [path.join(repo, 'doc.md')], - prompts: [judgePrompt], - provider, - workspaceRoot: repo, - scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], - outputFormat: OutputFormat.Json, - printMode: true, - sessionHomeDir: repo, - }); - - expect(result.hadOperationalErrors).toBe(false); - expect(result.findings).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - file: 'doc.md', - ruleId: 'Default.Consistency', - message: 'Use clearer wording', - }), - ]) - ); - }); - it('redacts raw read_file content from persisted tool_call_finished events', async () => { const { runAgentExecutor } = await import('../../src/agent/executor'); diff --git a/tests/evaluations/test-rules/Test/clarity.md b/tests/evaluations/test-rules/Test/clarity.md index b421b273..372b0688 100644 --- a/tests/evaluations/test-rules/Test/clarity.md +++ b/tests/evaluations/test-rules/Test/clarity.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Clarity name: Clarity severity: warning diff --git a/tests/evaluations/test-rules/Test/consistency.md b/tests/evaluations/test-rules/Test/consistency.md index 38ce7297..248d903b 100644 --- a/tests/evaluations/test-rules/Test/consistency.md +++ b/tests/evaluations/test-rules/Test/consistency.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Consistency name: Consistency severity: warning diff --git a/tests/evaluations/test-rules/Test/passive-voice.md b/tests/evaluations/test-rules/Test/passive-voice.md index bb714965..0df1eda6 100644 --- a/tests/evaluations/test-rules/Test/passive-voice.md +++ b/tests/evaluations/test-rules/Test/passive-voice.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: PassiveVoice name: Passive Voice severity: warning diff --git a/tests/evaluations/test-rules/Test/readability.md b/tests/evaluations/test-rules/Test/readability.md index e8806b38..e943f04b 100644 --- a/tests/evaluations/test-rules/Test/readability.md +++ b/tests/evaluations/test-rules/Test/readability.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Readability name: Readability severity: warning diff --git a/tests/evaluations/test-rules/Test/wordiness.md b/tests/evaluations/test-rules/Test/wordiness.md index f68106ab..77b42f41 100644 --- a/tests/evaluations/test-rules/Test/wordiness.md +++ b/tests/evaluations/test-rules/Test/wordiness.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Wordiness name: Wordiness severity: warning diff --git a/tests/evaluator.test.ts b/tests/evaluator.test.ts index b138a20d..79f0bb91 100644 --- a/tests/evaluator.test.ts +++ b/tests/evaluator.test.ts @@ -21,11 +21,11 @@ function setupEnv() { // Two prompts with minimal frontmatter criteria writeFileSync( path.join(promptsDir, 'p1.md'), - `---\nid: P1\nname: Prompt 1\ncriteria:\n - name: A\n id: A\n weight: 1\n---\nBody 1\n` + `---\nid: P1\nname: Prompt 1\ncriteria:\n - name: A\n id: A\n---\nBody 1\n` ); writeFileSync( path.join(promptsDir, 'p2.md'), - `---\nid: P2\nname: Prompt 2\ncriteria:\n - name: B\n id: B\n weight: 1\n---\nBody 2\n` + `---\nid: P2\nname: Prompt 2\ncriteria:\n - name: B\n id: B\n---\nBody 2\n` ); const files = [ path.join(root, 'a.md'), diff --git a/tests/findings/README.md b/tests/findings/README.md deleted file mode 100644 index c856a2b4..00000000 --- a/tests/findings/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# Finding Processing — Test Coverage & Behavior Mapping - -This directory owns the regression coverage for the **shared objective -violation-check finding-processing pipeline** (`src/findings/`). It documents -which behaviors moved out of `src/cli/orchestrator.ts` (and adjacent output -tests) into `src/findings/`, which behaviors were **intentionally removed**, and -which behaviors are **out of scope** for this phase. - -The pipeline (`processFindings`) composes a single path: - -```text -candidate findings ──▶ verify evidence ──▶ filter ──▶ dedupe ──▶ score ──▶ severity ──▶ ReviewResult -``` - -It is independent of model call (`single` / `agent` / `auto`) and carries no -`judge`, `evaluator`, rubric, or autonomous-agent compatibility path. - ---- - -## Behavior-preservation mapping - -| Behavior (origin) | Preserved by | Notes | -|---|---|---| -| Violation filtering / surfacing (`orchestrator.ts` `computeFilterDecision`) | `processor.test.ts` — *drops candidates that fail computeFilterDecision without a diagnostic*; *filtered candidates do not contribute to the score count* | Filter logic itself stays in `src/debug/violation-filter.ts`; the processor is a caller. `orchestrator-filtering.test.ts` still guards the end-to-end CLI surfacing. | -| Fuzzy finding-evidence location (`locateQuotedText`) | `finding-evidence-verifier.test.ts` — exact-quote location, warn diagnostic on miss, **no model-line fallback**, anchors without a line hint, diagnostic names the quoted text | One verifier (audit Finding #6). The agent path's model-line fallback is **not** preserved (see Removed). | -| Verified-finding counting & density scoring (`calculateCheckScore`) | `scorer.test.ts` — same numeric result as `calculateCheckScore`; score driven by verified count, not raw candidate count; resolves error severity; forwards strictness/severity | `scorer.ts` is a thin wrapper; no reimplemented math. | -| Severity + rule-id attribution | `severity.test.ts` — `resolveSeverity`, `buildRuleId` (`Pack.Rule` / `Pack.Rule.Criterion`), `resolveCriterionId`; `processor.test.ts` — stamps severity, attributes to pack rule / criterion | One severity path; one rule-id builder (audit Finding #4). | -| Golden check output (byte-for-byte) | `processor.test.ts` — *produces byte-for-byte findings/score matching the standard orchestrator path* | Proves no regression before the orchestrator was rewired (Task 2). | -| Unanchored-evidence diagnostics & exit behavior | `processor.test.ts` — *turns unanchored evidence into a warn diagnostic and emits no finding*; *counts only verified findings toward the score*; *does not set hadOperationalErrors for warn-level evidence diagnostics* | Intentional Phase 3 fix: unanchored quotes no longer flag the run as operationally failed. | -| Deduplication | `processor.test.ts` — *deduplicates verified findings by quoted_text and line* | | -| Contract / input boundary | `types.test.ts` — `FINDING_PROCESSING_INPUT_SCHEMA`, `RAW_VIOLATION_SCHEMA`, `PROMPT_META_FOR_FINDINGS_SCHEMA` parse supported input and reject legacy `evaluator`/`judge`/`rubric weight`/`agent`/`modelCall`/`mode`; `module-surface.test.ts` — public surface + diagnostic code constant | Strict Zod at the boundary; no `any`. | -| Formatter routing (line / JSON / RDJSON / Vale) | `orchestrator-check-processor.test.ts` — locatable findings + score + counts unchanged; verified findings through the JSON sink; unanchored quotes omitted from Vale JSON. `orchestrator-filtering.test.ts` — no dummy JSON/Vale issues when nothing is surfaced | End-to-end coverage of the rewired check path feeding existing formatters from `ReviewResult`. | - ---- - -## Intentionally removed behavior (no replacement fixture) - -These were legacy paths that the Phase 3 contract no longer supports. They are -covered by **rejection tests**, not behavior-parity tests. - -| Removed behavior | Rejection coverage | -|---|---| -| Subjective `judge` / rubric criterion evaluation | `prompt-loader-validation.test.ts` — *Judge boundary rejection (Phase 3)* rejects `type: judge` and the `subjective` alias at the loader boundary; `orchestrator-filtering.test.ts` — *rejects judge/rubric results instead of projecting them as check findings* | -| Legacy `evaluator` / `judge` / rubric criteria in the review contract | `review/types.test.ts` — *rejects legacy evaluator and judge criteria fields*; `findings/types.test.ts` — input contract rejects judge/evaluator/rubric shapes | -| Agent finding-evidence model-line fallback | `finding-evidence-verifier.test.ts` — *never returns a finding whose line came from the model when the quote did not anchor* (the removed behavior is asserted as absent) | - -Judge criterion output behavior (1–4 rubric, weighted average) has **no** -replacement fixture in this phase. Objective rule coverage belongs to the -objective-check test inventory, not a ported rubric. - ---- - -## Out of scope for Phase 3 (Phase 4 territory) - -These are **not** behavior-preservation targets for this phase and are -intentionally left untouched: - -- `tests/agent/*`, the orchestrator test covering legacy agent output, and - `tests/providers/*agent-loop*.test.ts` — the unreleased internal autonomous agent implementation. - Phase 4 deletes or replaces these with agent-model-call executor tests. -- `BaseEvaluator`'s internal judge mode (`runJudgeEvaluation`), - `buildJudgeLLMSchema` (`src/prompts/schema.ts`), the judge scoring helpers in - `src/scoring/`, `tests/scoring-types.test.ts`, and the judge cases in - `tests/prompt-schema.test.ts` — evaluator internals that are now **unreachable - from the CLI** (judge prompts fail to load) but are scheduled for deletion in - Phase 4 (executors / provider interface / agent deletion). - -Phase 3 added **no** compatibility code for the unreleased/internal agent implementation — no -agent tool-loop wiring, no review-instruction handler, and no agent result -projection — to `src/findings/`. - ---- - -## Files - -| File | Covers | -|---|---| -| `finding-evidence-verifier.test.ts` | Single evidence verifier; no model-line fallback. | -| `severity.test.ts` | `resolveSeverity`, `buildRuleId`, `resolveCriterionId`. | -| `scorer.test.ts` | `scoreCheck` numerics vs `calculateCheckScore`; verified-count scoring. | -| `types.test.ts` | Zod input contract: parses supported input, rejects legacy shapes. | -| `processor.test.ts` | `processFindings` golden output, diagnostics, counting fix, dedupe, rule-id attribution, `ReviewResult` validation. | -| `module-surface.test.ts` | Public barrel surface + stable diagnostic code constant. | diff --git a/tests/findings/finding-evidence-verifier.test.ts b/tests/findings/finding-evidence-verifier.test.ts index 28422d16..87ade746 100644 --- a/tests/findings/finding-evidence-verifier.test.ts +++ b/tests/findings/finding-evidence-verifier.test.ts @@ -20,7 +20,7 @@ describe('verifyFindingEvidence', () => { expect(result.diagnostic).toBeUndefined(); }); - it('returns a warn diagnostic (not a fallback finding) when evidence cannot be located', () => { + it('returns a warn diagnostic when evidence cannot be located', () => { const result = verifyFindingEvidence(CONTENT, { quoted_text: 'this does not exist anywhere', context_before: '', @@ -33,21 +33,6 @@ describe('verifyFindingEvidence', () => { expect(result.diagnostic?.level).toBe('warn'); }); - it('never returns a finding whose line came from the model when the quote did not anchor', () => { - // The unreleased internal agent implementation fell back to the - // model-provided line (here 1) and surfaced the raw quoted_text as a - // verified match. Assert we do not. - // (Use a lexically distant quote that locateQuotedText cannot anchor.) - const result = verifyFindingEvidence(CONTENT, { - quoted_text: 'definitely not present xyzzy', - context_before: '', - context_after: '', - line: 1, - }); - expect(result.verified).toBe(false); - expect(result.finding).toBeUndefined(); - }); - it('still anchors a quote when no line hint is provided', () => { const result = verifyFindingEvidence(CONTENT, { quoted_text: 'line three', @@ -64,7 +49,6 @@ describe('verifyFindingEvidence', () => { }); expect(result.verified).toBe(false); expect(result.diagnostic?.message).toContain('x'.repeat(60)); - // Truncated, not the full 120 chars. expect(result.diagnostic?.message).not.toContain('x'.repeat(61)); }); }); diff --git a/tests/findings/module-surface.test.ts b/tests/findings/module-surface.test.ts index a0e5c481..7d7661e0 100644 --- a/tests/findings/module-surface.test.ts +++ b/tests/findings/module-surface.test.ts @@ -8,7 +8,7 @@ describe('src/findings public surface', () => { expect(typeof findings.resolveSeverity).toBe('function'); expect(typeof findings.buildRuleId).toBe('function'); expect(typeof findings.resolveCriterionId).toBe('function'); - expect(typeof findings.scoreCheck).toBe('function'); + expect(typeof findings.scoreFindings).toBe('function'); }); it('exports the stable diagnostic code constant', () => { diff --git a/tests/findings/processor.test.ts b/tests/findings/processor.test.ts index 90f9641d..6b09e971 100644 --- a/tests/findings/processor.test.ts +++ b/tests/findings/processor.test.ts @@ -39,8 +39,8 @@ function baseInput( ): FindingProcessingInput { return { pack: 'TestPack', - ruleId: 'CheckPrompt', - ruleSource: '/repo/prompts/CheckPrompt.md', + ruleId: 'RulePrompt', + ruleSource: '/repo/prompts/RulePrompt.md', candidateFindings, wordCount: 100, promptMeta: { severity: 'warning', strictness: 'standard' }, @@ -53,7 +53,6 @@ describe('processFindings', () => { const originalThreshold = process.env.CONFIDENCE_THRESHOLD; beforeEach(() => { - // Default threshold (0.75) unless a test opts in. delete process.env.CONFIDENCE_THRESHOLD; }); @@ -65,13 +64,12 @@ describe('processFindings', () => { } }); - describe('golden objective-check output', () => { + describe('finding output', () => { beforeEach(() => { - // Surface every gate-passing candidate regardless of confidence. process.env.CONFIDENCE_THRESHOLD = '0.0'; }); - it('produces byte-for-byte findings/score matching the standard orchestrator path', () => { + it('produces verified findings and a density score', () => { const input = baseInput([ violation(), violation({ @@ -86,10 +84,9 @@ describe('processFindings', () => { const result = processFindings(input); - // Score: density 2/100 at standard strictness (10) -> 100 - 20 -> 8.0/10. expect(result.scores).toEqual([ { - ruleId: 'TestPack.CheckPrompt', + ruleId: 'TestPack.RulePrompt', score: 8.0, scoreText: '8.0/10', severity: 'warning', @@ -97,12 +94,10 @@ describe('processFindings', () => { }, ]); - // Findings: Pack.Rule (check violations carry no criterion name), - // anchored lines/columns/match. expect(result.findings).toHaveLength(2); expect(result.findings[0]).toMatchObject({ - ruleId: 'TestPack.CheckPrompt', - ruleSource: '/repo/prompts/CheckPrompt.md', + ruleId: 'TestPack.RulePrompt', + ruleSource: '/repo/prompts/RulePrompt.md', severity: 'warning', message: 'Issue 1', line: 1, @@ -112,7 +107,7 @@ describe('processFindings', () => { fix: 'Fix 1', }); expect(result.findings[1]).toMatchObject({ - ruleId: 'TestPack.CheckPrompt', + ruleId: 'TestPack.RulePrompt', severity: 'warning', message: 'Issue 2', line: 2, @@ -130,7 +125,7 @@ describe('processFindings', () => { }); }); - describe('evidence verification (audit Finding #6)', () => { + describe('evidence verification', () => { beforeEach(() => { process.env.CONFIDENCE_THRESHOLD = '0.0'; }); @@ -146,14 +141,11 @@ describe('processFindings', () => { expect(result.diagnostics).toHaveLength(1); expect(result.diagnostics[0]?.code).toBe(FINDING_EVIDENCE_NOT_LOCATABLE); expect(result.diagnostics[0]?.level).toBe('warn'); - // Score reflects 0 verified findings -> perfect 10.0/10. expect(result.scores[0]?.score).toBe(10.0); expect(result.scores[0]?.findingCount).toBe(0); }); it('counts only verified findings toward the score, not raw candidate count', () => { - // One anchored + one unanchored candidate. Verified count = 1. - // Density 1/100 -> 100 - 10 -> 9.0/10 (the intentional fix). const result = processFindings( baseInput([ violation({ quoted_text: 'Alpha text' }), @@ -190,7 +182,6 @@ describe('processFindings', () => { describe('candidate filtering', () => { it('drops candidates that fail computeFilterDecision without a diagnostic', () => { - // Default threshold 0.75: confidence 0.2 is filtered out. const result = processFindings( baseInput([ violation({ quoted_text: 'Alpha text', confidence: 0.2 }), @@ -199,7 +190,6 @@ describe('processFindings', () => { expect(result.findings).toEqual([]); expect(result.diagnostics).toEqual([]); - // No verified findings -> 10.0/10. expect(result.scores[0]?.findingCount).toBe(0); }); @@ -224,8 +214,8 @@ describe('processFindings', () => { it('attributes findings to Pack.Rule when violations carry no criterion name', () => { const result = processFindings(baseInput([violation({ quoted_text: 'Alpha text' })])); - expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt'); - expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + expect(result.findings[0]?.ruleId).toBe('TestPack.RulePrompt'); + expect(result.scores[0]?.ruleId).toBe('TestPack.RulePrompt'); }); it('attributes findings to Pack.Rule.Criterion when a violation names a declared criterion', () => { @@ -246,15 +236,13 @@ describe('processFindings', () => { }, ), ); - expect(result.findings[0]?.ruleId).toBe('TestPack.CheckPrompt.Hedging'); - // The score stays at Pack.Rule (no criterion), matching the orchestrator. - expect(result.scores[0]?.ruleId).toBe('TestPack.CheckPrompt'); + expect(result.findings[0]?.ruleId).toBe('TestPack.RulePrompt.Hedging'); + expect(result.scores[0]?.ruleId).toBe('TestPack.RulePrompt'); }); }); describe('severity resolution', () => { it('resolves error severity from the density score and stamps it on findings', () => { - // 20 verified violations over 100 words -> 0.0/10 -> prompt severity error. const result = processFindings( baseInput( Array.from({ length: 20 }, (_, i) => diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts index 7e8147b0..c7c7d19b 100644 --- a/tests/findings/scorer.test.ts +++ b/tests/findings/scorer.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest'; -import { calculateCheckScore } from '../../src/scoring'; +import { calculateScore } from '../../src/scoring'; import { Severity } from '../../src/evaluators/types'; -import { scoreCheck } from '../../src/findings/scorer'; +import { scoreFindings } from '../../src/findings/scorer'; import type { RawViolation } from '../../src/findings/types'; const FULLY_SUPPORTED_CHECKS = { @@ -25,17 +25,17 @@ function violation(overrides: Partial = {}): RawViolation { }; } -describe('scoreCheck', () => { - it('returns the same numeric result as calculateCheckScore for the same density', () => { +describe('scoreFindings', () => { + it('returns the same numeric result as calculateScore for the same density', () => { const verified = [violation(), violation()]; - const result = scoreCheck({ + const result = scoreFindings({ verifiedViolations: verified, wordCount: 100, strictness: 'standard', promptSeverity: Severity.WARNING, }); - const direct = calculateCheckScore(verified, 100, { + const direct = calculateScore(verified, 100, { strictness: 'standard', promptSeverity: Severity.WARNING, }); @@ -48,9 +48,7 @@ describe('scoreCheck', () => { }); it('is driven by the verified finding count, not a raw candidate count', () => { - // 1 verified violation over 100 words at standard strictness: - // density 1 -> 100 - 1*10 = 90 -> 9.0/10 - const one = scoreCheck({ + const one = scoreFindings({ verifiedViolations: [violation()], wordCount: 100, strictness: 'standard', @@ -58,8 +56,7 @@ describe('scoreCheck', () => { expect(one.score).toBe(9.0); expect(one.scoreText).toBe('9.0/10'); - // 2 verified violations -> density 2 -> 100 - 2*10 = 80 -> 8.0/10 - const two = scoreCheck({ + const two = scoreFindings({ verifiedViolations: [violation(), violation()], wordCount: 100, strictness: 'standard', @@ -69,9 +66,7 @@ describe('scoreCheck', () => { }); it('resolves error severity from the density score when score is low and prompt severity is error', () => { - // 20 violations over 100 words at standard strictness: - // density 20 -> 100 - 20*10 = -100 -> clamp 0 -> 0.0/10 -> prompt severity - const result = scoreCheck({ + const result = scoreFindings({ verifiedViolations: Array.from({ length: 20 }, () => violation()), wordCount: 100, strictness: 'standard', @@ -81,14 +76,14 @@ describe('scoreCheck', () => { expect(result.severity).toBe(Severity.ERROR); }); - it('forwards strictness and prompt severity options to calculateCheckScore', () => { + it('forwards strictness and prompt severity options to calculateScore', () => { const verified = [violation()]; - const result = scoreCheck({ + const result = scoreFindings({ verifiedViolations: verified, wordCount: 100, strictness: 'strict', }); - const direct = calculateCheckScore(verified, 100, { strictness: 'strict' }); + const direct = calculateScore(verified, 100, { strictness: 'strict' }); expect(result.score).toBe(direct.final_score); }); }); diff --git a/tests/findings/severity.test.ts b/tests/findings/severity.test.ts index 46e0c93a..031115f1 100644 --- a/tests/findings/severity.test.ts +++ b/tests/findings/severity.test.ts @@ -4,12 +4,11 @@ import { resolveCriterionId, resolveSeverity, } from '../../src/findings/severity'; -import { EvaluationType, Severity } from '../../src/evaluators/types'; -import type { CheckResult } from '../../src/prompts/schema'; +import { Severity } from '../../src/evaluators/types'; +import type { ScoredEvaluation } from '../../src/prompts/schema'; -function makeCheckResult(severity: Severity): CheckResult { +function makeScoredEvaluation(severity: Severity): ScoredEvaluation { return { - type: EvaluationType.CHECK, final_score: 5, percentage: 50, violation_count: 1, @@ -21,11 +20,11 @@ function makeCheckResult(severity: Severity): CheckResult { } describe('resolveSeverity', () => { - it('returns the density-derived severity from the check score result', () => { - expect(resolveSeverity({ scored: makeCheckResult(Severity.WARNING) })).toBe( + it('returns the density-derived severity from the scored evaluation', () => { + expect(resolveSeverity({ scored: makeScoredEvaluation(Severity.WARNING) })).toBe( Severity.WARNING, ); - expect(resolveSeverity({ scored: makeCheckResult(Severity.ERROR) })).toBe( + expect(resolveSeverity({ scored: makeScoredEvaluation(Severity.ERROR) })).toBe( Severity.ERROR, ); }); diff --git a/tests/findings/types.test.ts b/tests/findings/types.test.ts index 4309ead6..d55073e3 100644 --- a/tests/findings/types.test.ts +++ b/tests/findings/types.test.ts @@ -60,28 +60,11 @@ describe('RAW_VIOLATION_SCHEMA', () => { }); describe('PROMPT_META_FOR_FINDINGS_SCHEMA', () => { - it('rejects legacy evaluator/type/evaluateAs/target fields', () => { + it('rejects unknown fields', () => { expect(() => PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ severity: 'warning', - evaluator: 'base', - }), - ).toThrow(); - expect(() => - PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ type: 'judge' }), - ).toThrow(); - expect(() => - PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ evaluateAs: 'document' }), - ).toThrow(); - expect(() => - PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ target: { regex: 'x' } }), - ).toThrow(); - }); - - it('rejects rubric weight on criteria', () => { - expect(() => - PROMPT_META_FOR_FINDINGS_SCHEMA.parse({ - criteria: [{ id: 'Hedging', name: 'Hedge words', weight: 2 }], + unexpected: true, }), ).toThrow(); }); @@ -95,39 +78,6 @@ describe('FINDING_PROCESSING_INPUT_SCHEMA', () => { expect(parsed.promptMeta.criteria?.[0]?.id).toBe('Hedging'); }); - it('has no modelCall/mode/agent field and rejects them', () => { - for (const legacyKey of ['modelCall', 'mode', 'agent'] as const) { - expect(() => - FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), [legacyKey]: 'agent' }), - ).toThrow(); - } - }); - - it('rejects legacy judge/evaluator/rubric-shaped top-level input', () => { - // Legacy judge result shape: criteria array with scores at the top level. - expect(() => - FINDING_PROCESSING_INPUT_SCHEMA.parse({ - pack: 'VectorLint', - ruleId: 'Tone', - ruleSource: '/x.md', - wordCount: 100, - targetContent: 'x', - promptMeta: {}, - evaluator: 'judge', - criteria: [{ name: 'Tone', score: 2, violations: [] }], - }), - ).toThrow(); - }); - - it('rejects a legacy evaluator field nested under promptMeta', () => { - expect(() => - FINDING_PROCESSING_INPUT_SCHEMA.parse({ - ...validInput(), - promptMeta: { ...validInput().promptMeta, evaluator: 'base' }, - }), - ).toThrow(); - }); - it('rejects malformed candidate findings', () => { expect(() => FINDING_PROCESSING_INPUT_SCHEMA.parse({ @@ -139,7 +89,7 @@ describe('FINDING_PROCESSING_INPUT_SCHEMA', () => { it('rejects an unknown top-level key', () => { expect(() => - FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), judge: true }), + FINDING_PROCESSING_INPUT_SCHEMA.parse({ ...validInput(), unexpected: true }), ).toThrow(); }); }); diff --git a/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md b/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md index ab648b00..bdb8cae5 100644 --- a/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md +++ b/tests/fixtures/capitalization/rules/capitalization-rules/capitalization.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Capitalization name: Capitalization severity: warning @@ -9,4 +8,4 @@ evaluateAs: document # Capitalization -Flag any instance where headings within the same document are inconsistent in their capitalization style. Identify the dominant style used across headings — either title case (e.g., "Getting Started With the API") or sentence case (e.g., "Getting started with the API") — and flag any heading that deviates from it. For title case, also flag when the document mixes AP and Chicago conventions, such as capitalizing prepositions in some headings but not others (e.g., "Working With Your Team" vs. "Working with Your Team"). Do not flag proper nouns or product names that are always capitalized regardless of heading style. \ No newline at end of file +Flag any instance where headings within the same document are inconsistent in their capitalization style. Identify the dominant style used across headings — either title case (e.g., "Getting Started With the API") or sentence case (e.g., "Getting started with the API") — and flag any heading that deviates from it. For title case, also flag when the document mixes AP and Chicago conventions, such as capitalizing prepositions in some headings but not others (e.g., "Working With Your Team" vs. "Working with Your Team"). Do not flag proper nouns or product names that are always capitalized regardless of heading style. diff --git a/tests/fixtures/consistency/rules/consistency-rules/consistency.md b/tests/fixtures/consistency/rules/consistency-rules/consistency.md index ba038b36..219d1022 100644 --- a/tests/fixtures/consistency/rules/consistency-rules/consistency.md +++ b/tests/fixtures/consistency/rules/consistency-rules/consistency.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Consistency name: Consistency severity: warning diff --git a/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md b/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md index 5075f6d3..ab5bda8f 100644 --- a/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md +++ b/tests/fixtures/inclusivity/rules/inclusivity-rules/inclusivity.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Inclusivity name: Inclusivity severity: warning diff --git a/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md b/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md index fc13a7b3..bfa21666 100644 --- a/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md +++ b/tests/fixtures/passive-voice/rules/passive-voice-rules/passive-voice.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: PassiveVoice name: Passive Voice severity: warning @@ -11,4 +10,4 @@ evaluateAs: document Flag any sentence that uses passive voice in a way that makes the writing less clear, less direct, or more evasive — for example, when the doer is omitted and that omission reduces accountability or clarity ("Mistakes were made"), or when a modal passive weakens an instruction ("The file should be saved before closing"). -Do not flag passive voice when the sentence is describing a state, condition, or feature rather than an action; when the doer is obvious, irrelevant, or already established in the immediately preceding sentence; or when an active rewrite would be awkward or less natural than the original. \ No newline at end of file +Do not flag passive voice when the sentence is describing a state, condition, or feature rather than an action; when the doer is obvious, irrelevant, or already established in the immediately preceding sentence; or when an active rewrite would be awkward or less natural than the original. diff --git a/tests/fixtures/repetition/rules/repetition-rules/repetition.md b/tests/fixtures/repetition/rules/repetition-rules/repetition.md index 273ea83e..d102e343 100644 --- a/tests/fixtures/repetition/rules/repetition-rules/repetition.md +++ b/tests/fixtures/repetition/rules/repetition-rules/repetition.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Repetition name: Repetition severity: warning @@ -8,4 +7,4 @@ evaluateAs: document --- # Repetition -Flag any instance where the same word appears consecutively or near-consecutively in a sentence (e.g., "the the build" or "you can can also"), or where the same idea is expressed in more than one sentence within the same section without adding new information (e.g., "This speeds up your workflow. It helps teams work faster and get more done."). Do not flag repeated product names, feature names, or technical terms where varying the language would introduce ambiguity (e.g., "Components" repeated throughout a section about components), and do not flag repetition across structurally independent elements like cards, list items, or callouts that are designed to be read on their own (e.g., a card description that echoes a phrase from the body text it links to). \ No newline at end of file +Flag any instance where the same word appears consecutively or near-consecutively in a sentence (e.g., "the the build" or "you can can also"), or where the same idea is expressed in more than one sentence within the same section without adding new information (e.g., "This speeds up your workflow. It helps teams work faster and get more done."). Do not flag repeated product names, feature names, or technical terms where varying the language would introduce ambiguity (e.g., "Components" repeated throughout a section about components), and do not flag repetition across structurally independent elements like cards, list items, or callouts that are designed to be read on their own (e.g., a card description that echoes a phrase from the body text it links to). diff --git a/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md b/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md index 3115c6f3..07c74cff 100644 --- a/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md +++ b/tests/fixtures/unsupported-claims/rules/unsupported-claims-rules/unsupported-claims.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: UnsupportedClaims name: Unsupported Claims severity: warning @@ -12,4 +11,4 @@ Flag any claim that states a specific number, percentage, comparison, or causal Also flag vague authority citations where the source is unnamed (e.g., "experts say," "research shows," "it is well established"). -Do not flag claims the author makes about their own product or organization in the first person (e.g., "we processed over 1M requests last month," "our platform supports 40+ integrations"), as these are self-reported and do not require a third-party source. \ No newline at end of file +Do not flag claims the author makes about their own product or organization in the first person (e.g., "we processed over 1M requests last month," "our platform supports 40+ integrations"), as these are self-reported and do not require a third-party source. diff --git a/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md b/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md index d926bcfb..cc8529fc 100644 --- a/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md +++ b/tests/fixtures/wordiness/rules/wordiness-rules/wordiness.md @@ -1,6 +1,5 @@ --- evaluator: base -type: check id: Wordiness name: Wordiness severity: warning @@ -9,4 +8,4 @@ evaluateAs: document # Wordiness -Flag any instance where a wordy phrase is used in place of a shorter, simpler equivalent (e.g., "in order to" instead of "to", "due to the fact that" instead of "because", "has the ability to" instead of "can", "at this point in time" instead of "at this point"). This includes multi-word phrases that can be reduced to a single word, roundabout constructions that obscure a simpler verb or preposition, and redundant pairs where one word carries the full meaning (e.g., "gather together" instead of "gather", "close down" instead of "close"). Do not flag phrases where the longer form is required for clarity or where the substitution would change the meaning. \ No newline at end of file +Flag any instance where a wordy phrase is used in place of a shorter, simpler equivalent (e.g., "in order to" instead of "to", "due to the fact that" instead of "because", "has the ability to" instead of "can", "at this point in time" instead of "at this point"). This includes multi-word phrases that can be reduced to a single word, roundabout constructions that obscure a simpler verb or preposition, and redundant pairs where one word carries the full meaning (e.g., "gather together" instead of "gather", "close down" instead of "close"). Do not flag phrases where the longer form is required for clarity or where the substitution would change the meaning. diff --git a/tests/orchestrator-agent-output.test.ts b/tests/orchestrator-agent-output.test.ts index b6c8e271..baf5ae98 100644 --- a/tests/orchestrator-agent-output.test.ts +++ b/tests/orchestrator-agent-output.test.ts @@ -20,7 +20,6 @@ function makePrompt(): PromptFile { meta: { id: name, name, - type: 'check', severity: Severity.WARNING, }, }; @@ -54,10 +53,6 @@ function makeStandardProvider(): StandardProviderSpies { return { provider, runPromptStructured, runAgentToolLoop }; } -// `--mode agent` is an unreleased internal path. These tests prove the CLI/evaluateFiles path no -// longer reaches the autonomous agent executor: it warns through the injected -// logger and falls back to standard evaluation. The retained agent executor -// code is covered directly by tests/agent/* and removed in Phase 4. describe('agent mode fallback', () => { const tempRepos: string[] = []; @@ -80,7 +75,7 @@ describe('agent mode fallback', () => { } }); - it('emits an internal-status warning through the injected logger and falls back to standard evaluation', async () => { + it('warns through the injected logger and falls back to standard evaluation', async () => { const repo = createTempRepo(); const file = path.join(repo, 'doc.md'); writeFileSync(file, 'bad phrase\n', 'utf8'); @@ -102,9 +97,8 @@ describe('agent mode fallback', () => { }); expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining('unreleased internal path'), + expect.stringContaining('falls back to standard evaluation'), ); - // Standard evaluation ran; the autonomous executor did not. expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(runPromptStructured).toHaveBeenCalled(); expect(result.totalFiles).toBe(1); @@ -131,7 +125,7 @@ describe('agent mode fallback', () => { logger, }); - expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('unreleased internal path')); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('falls back to standard evaluation')); expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(runPromptStructured).toHaveBeenCalled(); }); @@ -155,7 +149,6 @@ describe('agent mode fallback', () => { scanPaths: [{ pattern: '**/*.md', runRules: ['Default'], overrides: {} }], }); - // No fallback warning and no executor invocation in standard mode. expect(runAgentToolLoop).not.toHaveBeenCalled(); expect(result.totalFiles).toBe(1); }); diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index b4d2cf89..c2ce4926 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -4,18 +4,17 @@ import { tmpdir } from "os"; import path from "path"; import { evaluateFiles } from "../src/cli/orchestrator"; import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; +import { Severity } from "../src/evaluators/types"; import type { Result } from "../src/output/json-formatter"; import type { PromptFile } from "../src/prompts/prompt-loader"; import type { ValeOutput } from "../src/schemas/vale-responses"; -import type { JudgeResult, RawCheckResult } from "../src/prompts/schema"; +import type { PromptEvaluationResult } from "../src/prompts/schema"; const { EVALUATE_MOCK } = vi.hoisted(() => ({ EVALUATE_MOCK: vi.fn(), })); -type CheckViolation = RawCheckResult["violations"][number]; -type JudgeViolation = JudgeResult["criteria"][number]["violations"][number]; +type Violation = PromptEvaluationResult["violations"][number]; vi.mock("../src/evaluators/index", () => ({ createEvaluator: vi.fn(() => ({ @@ -63,18 +62,9 @@ const FULLY_SUPPORTED_CHECKS = { fix_preserves_meaning: true, } as const; -const EMPTY_CHECK_NOTES: JudgeViolation["check_notes"] = { - rule_supports_claim: "", - evidence_exact: "", - context_supports_violation: "", - plausible_non_violation: "", - fix_is_drop_in: "", - fix_preserves_meaning: "", -}; - -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { +function makeViolation( + overrides: Partial = {} +): Violation { return { line: 1, analysis: "Issue 1", @@ -90,60 +80,16 @@ function makeCheckViolation( }; } -function makeJudgeViolation( - overrides: Partial = {} -): JudgeViolation { - const { check_notes: checkNotesOverrides, ...rest } = overrides; - return { - line: 1, - quoted_text: "Alpha text", - context_before: "", - context_after: "", - description: "Issue 1", - analysis: "Issue 1", - suggestion: "Suggestion 1", - fix: "Fix 1", - rule_quote: "Rule quote", - checks: FULLY_SUPPORTED_CHECKS, - check_notes: { - ...EMPTY_CHECK_NOTES, - ...(checkNotesOverrides ?? {}), - }, - confidence: 0.9, - ...rest, - }; -} - -function makeCheckResult(params: { - violations: CheckViolation[]; +function makeResult(params: { + violations: Violation[]; wordCount?: number; -}): RawCheckResult { +}): PromptEvaluationResult { return { - type: EvaluationType.CHECK, violations: params.violations, word_count: params.wordCount ?? 100, }; } -function makeJudgeResult(violations: JudgeViolation[]): JudgeResult { - return { - type: EvaluationType.JUDGE, - final_score: 5, - criteria: [ - { - name: "Clarity", - weight: 1, - score: 2, - normalized_score: 5, - weighted_points: 1, - summary: "Needs work", - reasoning: "Reasoning", - violations, - }, - ], - }; -} - describe("CLI violation filtering", () => { const originalThreshold = process.env.CONFIDENCE_THRESHOLD; @@ -164,20 +110,19 @@ describe("CLI violation filtering", () => { vi.restoreAllMocks(); }); - it("filters low-confidence check violations from CLI counts by default", async () => { + it("filters low-confidence violations from CLI counts by default", async () => { const targetFile = createTempFile("Alpha text\nBeta text\n"); const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", + id: "RulePrompt", + name: "Rule Prompt", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation(), - makeCheckViolation({ + makeViolation(), + makeViolation({ line: 2, analysis: "Issue 2", suggestion: "Suggestion 2", @@ -197,10 +142,10 @@ describe("CLI violation filtering", () => { process.env.CONFIDENCE_THRESHOLD = "0.0"; EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation(), - makeCheckViolation({ + makeViolation(), + makeViolation({ line: 2, analysis: "Issue 2", suggestion: "Suggestion 2", @@ -219,19 +164,18 @@ describe("CLI violation filtering", () => { expect(zeroThresholdRun.totalWarnings).toBe(2); }); - it("does not mark severity error when no check violations are surfaced", async () => { + it("does not mark severity error when no violations are surfaced", async () => { const targetFile = createTempFile("Alpha text\n"); const prompt = createPrompt({ id: "CheckErrorPrompt", name: "Check Error Prompt", - type: "check", severity: Severity.ERROR, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ + makeViolation({ confidence: 0.2, }), ], @@ -247,9 +191,9 @@ describe("CLI violation filtering", () => { process.env.CONFIDENCE_THRESHOLD = "0.0"; EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ + makeViolation({ confidence: 0.2, }), ], @@ -265,28 +209,22 @@ describe("CLI violation filtering", () => { }); it("score reflects only surfaced violations, not filtered-out ones", async () => { - // 100-word file: 2 violations from model, 1 fails confidence gate - // With default threshold, only 1 violation surfaces - // Density: 1/100 * 100 * 10 = 10 penalty → score = 9.0 - // If bug were present (scoring all 2): 2/100 * 100 * 10 = 20 penalty → score = 8.0 - const content = new Array(100).fill("word").join(" ") + "\n"; const targetFile = createTempFile(content); const prompt = createPrompt({ id: "ScorePrompt", name: "Score Prompt", - type: "check", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ quoted_text: content.split(" ")[0] ?? "word" }), - makeCheckViolation({ + makeViolation({ quoted_text: content.split(" ")[0] ?? "word" }), + makeViolation({ quoted_text: content.split(" ")[1] ?? "word", - confidence: 0.2, // fails confidence gate — should NOT affect score + confidence: 0.2, }), ], wordCount: 100, @@ -300,59 +238,24 @@ describe("CLI violation filtering", () => { await evaluateFiles([targetFile], createBaseOptions([prompt])); - // Score should reflect 1 surfaced violation, not 2 const scoreLine = logCalls.find(l => l.includes("/10")); expect(scoreLine).toBeDefined(); expect(scoreLine).toContain("9.0/10"); expect(scoreLine).not.toContain("8.0/10"); }); - it("rejects judge/rubric results instead of projecting them as check findings", async () => { - // Judge/rubric reviews are not a future-facing review type (Phase 3). The - // orchestrator refuses a JudgeResult rather than running the old criterion - // extraction/reporting path: no findings are projected and the run is - // flagged with an operational error. - const targetFile = createTempFile("Alpha text\nBeta text\n"); - const prompt = createPrompt({ - id: "JudgePrompt", - name: "Judge Prompt", - type: "judge", - criteria: [{ id: "Clarity", name: "Clarity", weight: 1 }], - severity: Severity.WARNING, - }); - - EVALUATE_MOCK.mockResolvedValue( - makeJudgeResult([ - makeJudgeViolation(), - makeJudgeViolation({ - line: 2, - quoted_text: "Beta text", - confidence: 0.2, - }), - ]) - ); - - const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(run.totalErrors).toBe(0); - expect(run.totalWarnings).toBe(0); - expect(run.hadOperationalErrors).toBe(true); - expect(run.hadSeverityErrors).toBe(false); - }); - it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { const targetFile = createTempFile("Alpha text\n"); const prompt = createPrompt({ id: "CheckJsonPrompt", name: "Check JSON Prompt", - type: "check", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ + makeViolation({ confidence: 0.2, }), ], @@ -380,14 +283,13 @@ describe("CLI violation filtering", () => { const prompt = createPrompt({ id: "CheckValeJsonPrompt", name: "Check Vale JSON Prompt", - type: "check", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ + makeViolation({ confidence: 0.2, }), ], diff --git a/tests/orchestrator-finding-criteria.test.ts b/tests/orchestrator-finding-criteria.test.ts deleted file mode 100644 index dc0b1ad8..00000000 --- a/tests/orchestrator-finding-criteria.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { mkdtempSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import path from "path"; -import { evaluateFiles } from "../src/cli/orchestrator"; -import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; -import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; -import type { ReviewResult } from "../src/review/types"; - -const { EVALUATE_MOCK, PROCESS_FINDINGS_MOCK } = vi.hoisted(() => ({ - EVALUATE_MOCK: vi.fn(), - PROCESS_FINDINGS_MOCK: vi.fn<(input: unknown) => ReviewResult>(), -})); - -// createEvaluator returns a controllable evaluator so evaluateFiles reaches -// routePromptResult -> processFindings without a real model call. -vi.mock("../src/evaluators/index", () => ({ - createEvaluator: vi.fn(() => ({ - evaluate: EVALUATE_MOCK, - })), -})); - -// Replace only processFindings (the findings boundary); keep every other -// findings export real so unrelated imports stay intact. -vi.mock("../src/findings", async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, processFindings: PROCESS_FINDINGS_MOCK }; -}); - -function createPrompt(meta: PromptFile["meta"]): PromptFile { - return { - id: meta.id, - filename: `${meta.id}.md`, - fullPath: path.join(process.cwd(), "prompts", `${meta.id}.md`), - meta, - body: "Prompt body", - pack: "TestPack", - }; -} - -function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { - return { - prompts, - rulesPath: undefined, - provider: {} as never, - concurrency: 1, - verbose: false, - debugJson: false, - scanPaths: [], - outputFormat: OutputFormat.Line, - }; -} - -function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-criteria-")); - const filePath = path.join(dir, "input.md"); - writeFileSync(filePath, content); - return filePath; -} - -function makeCheckResult(): RawCheckResult { - return { - type: EvaluationType.CHECK, - violations: [], - word_count: 100, - }; -} - -describe("standard check orchestration sanitizes criteria at the findings boundary", () => { - beforeEach(() => { - EVALUATE_MOCK.mockReset(); - PROCESS_FINDINGS_MOCK.mockReset(); - vi.spyOn(console, "log").mockImplementation(() => undefined); - vi.spyOn(console, "warn").mockImplementation(() => undefined); - vi.spyOn(console, "error").mockImplementation(() => undefined); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("drops rubric weight/target from criteria before reaching processFindings", async () => { - const targetFile = createTempFile("Alpha text\n"); - // PromptCriterionSpec admits legacy rubric fields (weight, target). The - // orchestrator must strip them so only { id, name } crosses the boundary. - const prompt = createPrompt({ - id: "SanitizePrompt", - name: "Sanitize Prompt", - type: "check", - severity: Severity.WARNING, - criteria: [ - { id: "Hedging", name: "Hedge words", weight: 3, target: { regex: "x" } }, - ], - }); - - EVALUATE_MOCK.mockResolvedValue(makeCheckResult()); - PROCESS_FINDINGS_MOCK.mockReturnValue({ - findings: [], - scores: [ - { - ruleId: "TestPack.SanitizePrompt", - score: 10, - scoreText: "10.0/10", - severity: "warning", - }, - ], - diagnostics: [], - }); - - await evaluateFiles([targetFile], createBaseOptions([prompt])); - - expect(PROCESS_FINDINGS_MOCK).toHaveBeenCalledTimes(1); - const input = PROCESS_FINDINGS_MOCK.mock.calls[0]![0] as { - promptMeta: { criteria?: Array> }; - }; - - // toEqual fails if any extra (weight/target) key survives the mapping. - expect(input.promptMeta.criteria).toEqual([{ id: "Hedging", name: "Hedge words" }]); - for (const criterion of input.promptMeta.criteria ?? []) { - expect(criterion).not.toHaveProperty("weight"); - expect(criterion).not.toHaveProperty("target"); - } - }); -}); diff --git a/tests/orchestrator-check-processor.test.ts b/tests/orchestrator-finding-processor.test.ts similarity index 77% rename from tests/orchestrator-check-processor.test.ts rename to tests/orchestrator-finding-processor.test.ts index eaaa1529..e15ef2ef 100644 --- a/tests/orchestrator-check-processor.test.ts +++ b/tests/orchestrator-finding-processor.test.ts @@ -4,17 +4,17 @@ import { tmpdir } from "os"; import path from "path"; import { evaluateFiles } from "../src/cli/orchestrator"; import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; -import { EvaluationType, Severity } from "../src/evaluators/types"; +import { Severity } from "../src/evaluators/types"; import type { Result } from "../src/output/json-formatter"; import type { ValeOutput } from "../src/schemas/vale-responses"; import type { PromptFile } from "../src/prompts/prompt-loader"; -import type { RawCheckResult } from "../src/prompts/schema"; +import type { PromptEvaluationResult } from "../src/prompts/schema"; const { EVALUATE_MOCK } = vi.hoisted(() => ({ EVALUATE_MOCK: vi.fn(), })); -type CheckViolation = RawCheckResult["violations"][number]; +type Violation = PromptEvaluationResult["violations"][number]; vi.mock("../src/evaluators/index", () => ({ createEvaluator: vi.fn(() => ({ @@ -56,20 +56,15 @@ function createBaseOptions(prompts: PromptFile[]): EvaluationOptions { } function createTempFile(content: string): string { - const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-check-proc-")); + const dir = mkdtempSync(path.join(tmpdir(), "vectorlint-finding-proc-")); const filePath = path.join(dir, "input.md"); writeFileSync(filePath, content); return filePath; } -/** - * Builds a check violation that passes the confidence gate by default. Real - * model output always carries `message`; the helper sets it explicitly so the - * test reflects production behavior rather than the analysis fallback. - */ -function makeCheckViolation( - overrides: Partial = {} -): CheckViolation { +function makeViolation( + overrides: Partial = {} +): Violation { return { line: 1, analysis: "Issue 1", @@ -86,18 +81,17 @@ function makeCheckViolation( }; } -function makeCheckResult(params: { - violations: CheckViolation[]; +function makeResult(params: { + violations: Violation[]; wordCount?: number; -}): RawCheckResult { +}): PromptEvaluationResult { return { - type: EvaluationType.CHECK, violations: params.violations, word_count: params.wordCount ?? 100, }; } -describe("standard check evaluation via the shared finding processor", () => { +describe("evaluation via the shared finding processor", () => { const originalThreshold = process.env.CONFIDENCE_THRESHOLD; beforeEach(() => { @@ -120,17 +114,16 @@ describe("standard check evaluation via the shared finding processor", () => { it("reports fully locatable findings, score, and counts unchanged", async () => { const targetFile = createTempFile("Alpha text\nBeta text\n"); const prompt = createPrompt({ - id: "CheckPrompt", - name: "Check Prompt", - type: "check", + id: "RulePrompt", + name: "Rule Prompt", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ message: "First issue", analysis: "First issue" }), - makeCheckViolation({ + makeViolation({ message: "First issue", analysis: "First issue" }), + makeViolation({ line: 2, quoted_text: "Beta text", message: "Second issue", @@ -150,7 +143,6 @@ describe("standard check evaluation via the shared finding processor", () => { const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - // Both quotes anchor: 2 verified findings over 100 words -> 8.0/10. expect(run.totalWarnings).toBe(2); expect(run.totalErrors).toBe(0); expect(run.hadOperationalErrors).toBe(false); @@ -158,26 +150,22 @@ describe("standard check evaluation via the shared finding processor", () => { const scoreLine = logCalls.find((l) => l.includes("/10")); expect(scoreLine).toBeDefined(); expect(scoreLine).toContain("8.0/10"); - // The line summary prints the last segment of the Pack.Rule score id. - expect(scoreLine).toContain("CheckPrompt"); + expect(scoreLine).toContain("RulePrompt"); }); it("counts only verified findings and excludes unanchored quotes from the score", async () => { - // One anchored + one unanchored quote. Both pass the confidence gate, so - // only the verifier distinguishes them. Verified count = 1 -> 9.0/10. const targetFile = createTempFile("Alpha text\n"); const prompt = createPrompt({ id: "CountingFixPrompt", name: "Counting Fix Prompt", - type: "check", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ + makeViolation({ quoted_text: "Alpha text" }), + makeViolation({ line: 9, quoted_text: "this quote is not anywhere in the content", message: "Ghost issue", @@ -195,8 +183,6 @@ describe("standard check evaluation via the shared finding processor", () => { const run = await evaluateFiles([targetFile], createBaseOptions([prompt])); - // Intentional Phase 3 fix: the unanchored quote is a diagnostic, not a - // finding; it contributes neither to the count nor the score. expect(run.totalWarnings).toBe(1); expect(run.hadOperationalErrors).toBe(false); @@ -206,20 +192,17 @@ describe("standard check evaluation via the shared finding processor", () => { }); it("does not flag severity errors when no finding can be verified", async () => { - // severity=error, but the only quote cannot anchor -> 0 verified findings -> - // perfect 10.0/10 -> severity resolves to warning -> no error exit signal. const targetFile = createTempFile("Alpha text\n"); const prompt = createPrompt({ id: "ErrorNoVerifiedPrompt", name: "Error No Verified Prompt", - type: "check", severity: Severity.ERROR, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ + makeViolation({ line: 9, quoted_text: "nowhere to be found", message: "Ghost", @@ -238,19 +221,16 @@ describe("standard check evaluation via the shared finding processor", () => { }); it("flags severity errors when verified error-severity findings exist", async () => { - // 1 verified finding over 10 words -> density 10 -> score 0.0 < 10 -> - // prompt severity error applies. const targetFile = createTempFile("Alpha text\n"); const prompt = createPrompt({ id: "ErrorVerifiedPrompt", name: "Error Verified Prompt", - type: "check", severity: Severity.ERROR, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ - violations: [makeCheckViolation({ quoted_text: "Alpha text" })], + makeResult({ + violations: [makeViolation({ quoted_text: "Alpha text" })], wordCount: 10, }) ); @@ -266,15 +246,14 @@ describe("standard check evaluation via the shared finding processor", () => { const prompt = createPrompt({ id: "CheckJsonPrompt", name: "Check JSON Prompt", - type: "check", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ message: "First", analysis: "First" }), - makeCheckViolation({ + makeViolation({ message: "First", analysis: "First" }), + makeViolation({ line: 2, quoted_text: "Beta text", message: "Second", @@ -316,15 +295,14 @@ describe("standard check evaluation via the shared finding processor", () => { const prompt = createPrompt({ id: "CheckValePrompt", name: "Check Vale Prompt", - type: "check", severity: Severity.WARNING, }); EVALUATE_MOCK.mockResolvedValue( - makeCheckResult({ + makeResult({ violations: [ - makeCheckViolation({ quoted_text: "Alpha text" }), - makeCheckViolation({ + makeViolation({ quoted_text: "Alpha text" }), + makeViolation({ line: 9, quoted_text: "missing quote", message: "Ghost", diff --git a/tests/prompt-loader-validation.test.ts b/tests/prompt-loader-validation.test.ts index fc34cc1d..80caac1a 100644 --- a/tests/prompt-loader-validation.test.ts +++ b/tests/prompt-loader-validation.test.ts @@ -23,7 +23,7 @@ describe('Prompt Loader Validation', () => { }); describe('Base Evaluator', () => { - it('should load base prompt with criteria (optional weight)', () => { + it('should load base prompt with criteria', () => { createPrompt('test.md', `--- evaluator: base id: Test @@ -54,24 +54,6 @@ Check content.`); expect(prompts).toHaveLength(1); }); - it('should load base prompt with weight in criteria', () => { - createPrompt('test.md', `--- -evaluator: base -id: Test -name: Test Evaluator -criteria: - - name: Quality - id: QualityCheck - weight: 1 ---- -Check content.`); - - const { prompts, warnings } = loadRules(tmpDir); - expect(warnings).toHaveLength(0); - expect(prompts).toHaveLength(1); - expect(prompts[0].meta.criteria![0].weight).toBe(1); - }); - it('should reject base prompt missing id', () => { createPrompt('test.md', `--- evaluator: base @@ -131,54 +113,20 @@ Check content.`); }); }); - describe('Judge boundary rejection (Phase 3)', () => { - it('rejects judge-typed prompts at the loader boundary', () => { - createPrompt('judge.md', `--- -evaluator: base -id: Judge -name: Judge Prompt -type: judge -criteria: - - name: Clarity - id: Clarity ---- -Judge content.`); - - const { prompts, warnings } = loadRules(tmpDir); - expect(prompts).toHaveLength(0); - expect(warnings.length).toBeGreaterThan(0); - expect(warnings[0]).toContain('judge.md'); - expect(warnings[0]).toMatch(/judge/i); - }); - - it('rejects subjective-typed prompts (deprecated judge alias)', () => { - createPrompt('subjective.md', `--- -evaluator: base -id: Subjective -name: Subjective Prompt -type: subjective ---- -Subjective content.`); - - const { prompts, warnings } = loadRules(tmpDir); - expect(prompts).toHaveLength(0); - expect(warnings.length).toBeGreaterThan(0); - expect(warnings[0]).toContain('subjective.md'); - }); - - it('still loads check-typed prompts', () => { - createPrompt('check.md', `--- + describe('ignored frontmatter', () => { + it('ignores a supplied type field', () => { + createPrompt('test.md', `--- evaluator: base -id: Check -name: Check Prompt -type: check +id: Test +name: Test Prompt +type: unused --- Check content.`); const { prompts, warnings } = loadRules(tmpDir); expect(warnings).toHaveLength(0); expect(prompts).toHaveLength(1); - expect(prompts[0].meta.type).toBe('check'); + expect(prompts[0].meta).not.toHaveProperty('type'); }); }); }); diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts index bf651a8f..383bad08 100644 --- a/tests/prompt-schema.test.ts +++ b/tests/prompt-schema.test.ts @@ -1,11 +1,10 @@ import { describe, expect, it } from "vitest"; -import { buildCheckLLMSchema, buildJudgeLLMSchema } from "../src/prompts/schema"; +import { buildEvaluationLLMSchema } from "../src/prompts/schema"; describe("prompt schema verbosity constraints", () => { - it("includes concise analysis and suggestion descriptions for check schema", () => { - const schema = buildCheckLLMSchema(); - const violationProperties = - schema.schema.properties.violations.items.properties; + it("includes concise analysis and suggestion descriptions", () => { + const schema = buildEvaluationLLMSchema(); + const violationProperties = schema.schema.properties.violations.items.properties; expect(violationProperties.analysis).toEqual({ type: "string", @@ -17,55 +16,14 @@ describe("prompt schema verbosity constraints", () => { }); }); - it("includes concise analysis and suggestion descriptions for judge schema", () => { - const schema = buildJudgeLLMSchema(); - const violationProperties = - schema.schema.properties.criteria.items.properties.violations.items - .properties; - - expect(violationProperties.analysis).toEqual({ - type: "string", - description: "A concise 1-2 sentence explanation of the specific issue.", - }); - expect(violationProperties.suggestion).toEqual({ - type: "string", - description: "Suggest a fix in 15 words or less.", - }); - }); - - it("check schema violation includes message field with correct description", () => { - const schema = buildCheckLLMSchema(); - const violationProperties = - schema.schema.properties.violations.items.properties; + it("includes the concise user-facing message field", () => { + const schema = buildEvaluationLLMSchema(); + const violationProperties = schema.schema.properties.violations.items.properties; expect(violationProperties.message).toEqual({ type: "string", description: "Under 15 words. State the issue directly to the user. No rule references.", }); - }); - - it("judge schema violation includes message field with correct description", () => { - const schema = buildJudgeLLMSchema(); - const violationProperties = - schema.schema.properties.criteria.items.properties.violations.items - .properties; - - expect(violationProperties.message).toEqual({ - type: "string", - description: "Under 15 words. State the issue directly to the user. No rule references.", - }); - }); - - it("check schema violation required array includes message", () => { - const schema = buildCheckLLMSchema(); - const required = schema.schema.properties.violations.items.required; - expect(required).toContain("message"); - }); - - it("judge schema violation required array includes message", () => { - const schema = buildJudgeLLMSchema(); - const required = - schema.schema.properties.criteria.items.properties.violations.items.required; - expect(required).toContain("message"); + expect(schema.schema.properties.violations.items.required).toContain("message"); }); }); diff --git a/tests/review/request-builder.test.ts b/tests/review/request-builder.test.ts index affb83a2..11381ac1 100644 --- a/tests/review/request-builder.test.ts +++ b/tests/review/request-builder.test.ts @@ -18,7 +18,6 @@ function makePrompt(overrides: Partial = {}): PromptFile { id: 'PseudoAdvice', name: 'Pseudo Advice', severity: Severity.WARNING, - type: 'check', criteria: [{ id: 'Vague', name: 'Vague advice' }], }, body: 'Check for pseudo advice.', diff --git a/tests/review/types.test.ts b/tests/review/types.test.ts index 3479074f..c8da35a5 100644 --- a/tests/review/types.test.ts +++ b/tests/review/types.test.ts @@ -43,7 +43,7 @@ describe('ReviewRule schema', () => { REVIEW_RULE_SCHEMA.parse({ id: 'Tone', source: 'VectorLint/tone.md', - body: 'Judge whether the tone is good.', + body: 'Determine whether the tone is good.', unexpected: true, }), ).toThrow(); diff --git a/tests/scoring-types.test.ts b/tests/scoring-types.test.ts index 7e9e018e..08ee7357 100644 --- a/tests/scoring-types.test.ts +++ b/tests/scoring-types.test.ts @@ -1,12 +1,8 @@ import { describe, it, expect, vi } from "vitest"; import { BaseEvaluator } from "../src/evaluators/base-evaluator"; -import { EvaluationType } from "../src/evaluators/types"; import type { LLMProvider, LLMResult } from "../src/providers/llm-provider"; import type { PromptFile } from "../src/schemas/prompt-schemas"; -import type { - JudgeLLMResult, - CheckLLMResult, -} from "../src/prompts/schema"; +import type { EvaluationLLMResult } from "../src/prompts/schema"; import type { SearchProvider } from "../src/providers/search-provider"; describe("Scoring Types", () => { @@ -14,88 +10,24 @@ describe("Scoring Types", () => { runPromptStructured: vi.fn(), } as unknown as LLMProvider; - describe("Judge Evaluation", () => { - const judgePrompt: PromptFile = { - id: "test-judge", + describe("Evaluation", () => { + const prompt: PromptFile = { + id: "test-rule", filename: "test.md", fullPath: "/test.md", - body: "Evaluate this.", + body: "Find violations.", pack: "test", meta: { - id: "test-judge", - name: "Test Judge", - type: "judge", - criteria: [ - { id: "c1", name: "Criterion 1", weight: 50 }, - { id: "c2", name: "Criterion 2", weight: 50 }, - ], - }, - }; - - it("should calculate weighted average correctly", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, judgePrompt); - - // Mock LLM returning raw scores (0-4) wrapped in LLMResult - const mockLlmResponse: LLMResult = { - data: { - criteria: [ - { - name: "Criterion 1", - score: 4, // 100% - summary: "Good", - reasoning: "Reason", - violations: [], - }, - { - name: "Criterion 2", - score: 2, // 50% - summary: "Okay", - reasoning: "Reason", - violations: [], - }, - ], - }, - }; - - // eslint-disable-next-line @typescript-eslint/unbound-method - const mockFn = vi.mocked(mockLlmProvider.runPromptStructured); - mockFn.mockResolvedValueOnce(mockLlmResponse); - - const result = await evaluator.evaluate("file.md", "content"); - - if (result.type !== EvaluationType.JUDGE) - throw new Error("Wrong result type"); - - // Calculation: - // C1: 10 (score 4) * 50 = 500 - // C2: 4 (score 2) * 50 = 200 - // Total: 700 / 100 = 7 - // Final Score: 7.0 - expect(result.final_score).toBe(7.0); - expect(result.criteria[0]!.weighted_points).toBe(500); - expect(result.criteria[1]!.weighted_points).toBe(200); - }); - }); - - describe("Check Evaluation", () => { - const checkPrompt: PromptFile = { - id: "test-check", - filename: "test.md", - fullPath: "/test.md", - body: "Count things.", - pack: "test", - meta: { - id: "test-check", - name: "Test Check", - type: "check", + id: "test-rule", + name: "Test Rule", }, }; it("should calculate score correctly based on violation count", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, checkPrompt); + const evaluator = new BaseEvaluator(mockLlmProvider, prompt); // Mock LLM returning violations only wrapped in LLMResult - const mockLlmResponse: LLMResult = { + const mockLlmResponse: LLMResult = { data: { violations: [ { @@ -125,18 +57,14 @@ describe("Scoring Types", () => { const content = new Array(100).fill("word").join(" "); const result = await evaluator.evaluate("file.md", content); - if (result.type !== EvaluationType.CHECK) - throw new Error("Wrong result type"); - - // Evaluator now returns raw violations and word count — scoring deferred to orchestrator expect(result.violations).toHaveLength(2); expect(result.word_count).toBe(100); }); it("should handle empty violations list (perfect score)", async () => { - const evaluator = new BaseEvaluator(mockLlmProvider, checkPrompt); + const evaluator = new BaseEvaluator(mockLlmProvider, prompt); - const mockLlmResponse: LLMResult = { + const mockLlmResponse: LLMResult = { data: { violations: [], }, @@ -148,9 +76,6 @@ describe("Scoring Types", () => { const result = await evaluator.evaluate("file.md", "content"); - if (result.type !== EvaluationType.CHECK) - throw new Error("Wrong result type"); - expect(result.violations).toHaveLength(0); expect(result.word_count).toBeGreaterThan(0); }); @@ -180,7 +105,7 @@ describe("Scoring Types", () => { fullPath: "/tech.md", body: "Check accuracy", pack: "test", - meta: { id: "tech-acc", name: "Tech Acc", type: "check" }, + meta: { id: "tech-acc", name: "Tech Acc" }, }; const evaluator = new TechnicalAccuracyEvaluator( @@ -196,8 +121,6 @@ describe("Scoring Types", () => { const result = await evaluator.evaluate("file.md", "content"); - if (result.type !== EvaluationType.CHECK) - throw new Error("Wrong result type"); expect(result.violations).toHaveLength(0); expect(result.word_count).toBeGreaterThan(0); }); diff --git a/tests/target.test.ts b/tests/target.test.ts index 08fe1a54..b99efa49 100644 --- a/tests/target.test.ts +++ b/tests/target.test.ts @@ -30,13 +30,10 @@ describe('Target gating (regex)', () => { "criteria:", " - name: A", " id: A", - " weight: 1", ].join('\n'); const { prompt } = setupPrompt(yaml); const content = 'No heading here'; - const c = prompt.meta.criteria?.[0]; - if (!c) throw new Error("No criteria"); - const res = checkTarget(content, prompt.meta.target, c.target); + const res = checkTarget(content, prompt.meta.target); expect(res.missing).toBe(true); expect(res.suggestion).toMatch(/H1/); }); @@ -52,43 +49,13 @@ describe('Target gating (regex)', () => { "criteria:", " - name: A", " id: A", - " weight: 1", ].join('\n'); const { prompt } = setupPrompt(yaml); const content = '# Title\n\nBody'; - const c = prompt.meta.criteria?.[0]; - if (!c) throw new Error("No criteria"); - const res = checkTarget(content, prompt.meta.target, c.target); + const res = checkTarget(content, prompt.meta.target); expect(res.missing).toBe(false); }); - it('criterion target overrides global target', () => { - const yaml = [ - "id: TestPrompt", - "name: Test Prompt", - "target:", - " regex: '^#\\s+(.+)$'", - " flags: 'mu'", - " required: true", - "criteria:", - " - name: A", - " id: A", - " weight: 1", - " target:", - " regex: '^##\\s+(.+)$'", - " flags: 'mu'", - " required: true", - " suggestion: Add an H2.", - ].join('\n'); - const { prompt } = setupPrompt(yaml); - const content = '# H1 only'; - const c = prompt.meta.criteria?.[0]; - if (!c) throw new Error("No criteria"); - const res = checkTarget(content, prompt.meta.target, c.target); - expect(res.missing).toBe(true); - expect(res.suggestion).toMatch(/H2/); - }); - it('invalid regex with required=true yields missing', () => { const yaml = [ "id: TestPrompt", @@ -100,13 +67,10 @@ describe('Target gating (regex)', () => { "criteria:", " - name: A", " id: A", - " weight: 1", ].join('\n'); const { prompt } = setupPrompt(yaml); const content = '# Title'; - const c = prompt.meta.criteria?.[0]; - if (!c) throw new Error("No criteria"); - const res = checkTarget(content, prompt.meta.target, c.target); + const res = checkTarget(content, prompt.meta.target); expect(res.missing).toBe(true); }); }); diff --git a/tests/validator.test.ts b/tests/validator.test.ts index 20dbf062..c695bfed 100644 --- a/tests/validator.test.ts +++ b/tests/validator.test.ts @@ -2,8 +2,8 @@ import { describe, it, expect } from 'vitest'; import { mkdtempSync, mkdirSync, writeFileSync } from 'fs'; import path from 'path'; import { tmpdir } from 'os'; -import { loadRules, type PromptFile } from '../src/prompts/prompt-loader.js'; -import { validateAll, validatePrompt } from '../src/prompts/prompt-validator.js'; +import { loadRules } from '../src/prompts/prompt-loader.js'; +import { validateAll } from '../src/prompts/prompt-validator.js'; function writePrompt(dir: string, name: string, yaml: string, body = 'Body') { const full = path.join(dir, name); @@ -22,10 +22,8 @@ describe('PromptValidator', () => { 'criteria:', ' - name: A', ' id: A', - ' weight: 2', ' - name: B', ' id: B', - ' weight: 2', ].join('\n'); writePrompt(promptsDir, 'ok.md', yaml); const { prompts } = loadRules(promptsDir); @@ -47,7 +45,6 @@ describe('PromptValidator', () => { 'criteria:', ' - name: A', ' id: A', - ' weight: 2', ].join('\n'); writePrompt(promptsDir, 'bad.md', yaml); const { prompts } = loadRules(promptsDir); @@ -55,24 +52,4 @@ describe('PromptValidator', () => { expect(res.errors.some(e => /Invalid regex flags/i.test(e.message))).toBe(true); expect(res.errors.some(e => /Invalid global target\.regex/i.test(e.message))).toBe(true); }); - - it('errors on invalid weights (manual prompt)', () => { - const p: PromptFile = { - id: 'x', - filename: 'x.md', - fullPath: '/path/to/x.md', - body: '', - meta: { - id: 'x', - name: 'X', - criteria: [ - { id: 'A', name: 'A', weight: 0 }, - { id: 'B', name: 'B', weight: -1 }, - ], - }, - }; - const res = validatePrompt(p); - const weightErrors = res.filter(e => /Invalid weight/i.test(e.message)); - expect(weightErrors.length).toBeGreaterThanOrEqual(2); - }); }); From f2da13fc79b9014b15f0068612251b85f5751caa Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Thu, 23 Jul 2026 01:14:05 +0100 Subject: [PATCH 24/24] refactor(vectorlint): harden finding processing boundaries - Deduplicate candidates by verified anchors and normalize empty targets - Share target regex defaults between validation and execution - Align strictness docs, evaluation types, and typed test fixtures --- CREATING_RULES.md | 8 ++++---- src/findings/processor.ts | 16 ++++++--------- src/findings/scorer.ts | 2 +- src/prompts/prompt-validator.ts | 3 ++- src/prompts/schema.ts | 18 +---------------- src/prompts/target.ts | 4 +++- tests/findings/processor.test.ts | 17 +++++++++++----- tests/findings/scorer.test.ts | 11 +++++++++++ tests/scoring-types.test.ts | 34 ++++++++++++++++++++++++++++++++ tests/validator.test.ts | 21 ++++++++++++++++++++ 10 files changed, 95 insertions(+), 39 deletions(-) diff --git a/CREATING_RULES.md b/CREATING_RULES.md index 7254cd25..d9c60cdf 100644 --- a/CREATING_RULES.md +++ b/CREATING_RULES.md @@ -156,10 +156,10 @@ target: | `evaluator` | string | No | Evaluator type: `base`, `technical-accuracy` (default: `base`) | | `id` | string | **Yes** | Unique identifier (used in error reporting) | | `name` | string | **Yes** | Human-readable name | - -| `severity` | string | No | `error` or `warning` (default: `warning`) | -| `evaluateAs` | string | No | `document` or `chunk` - whether to evaluate content as a whole or in chunks (default: `chunk`) | -| `target` | object | No | Content matching specification | +| `severity` | string | No | `error` or `warning` (default: `warning`) | +| `strictness` | number/string | No | Density penalty: a positive number, `lenient`, `standard`, or `strict` | +| `evaluateAs` | string | No | `document` or `chunk` (default: `chunk`) | +| `target` | object | No | Content matching specification | --- diff --git a/src/findings/processor.ts b/src/findings/processor.ts index d4f9c72f..af9a0641 100644 --- a/src/findings/processor.ts +++ b/src/findings/processor.ts @@ -94,8 +94,8 @@ function filterCandidates(candidates: readonly RawViolation[]): RawViolation[] { /** * Verifies each surfaced finding's evidence and deduplicates verified findings - * by `(quoted_text, line)`. Unanchored findings become warn diagnostics and are - * not emitted. + * by their anchored coordinates. Unanchored findings become warn diagnostics + * and are not emitted. */ function verifyAndDedupe( surfaced: readonly RawViolation[], @@ -129,15 +129,11 @@ function verifyAndDedupe( } const coords = verification.finding; - const dedupeKey = candidate.quoted_text - ? `${candidate.quoted_text}:${coords.line}` - : null; - if (dedupeKey) { - if (seen.has(dedupeKey)) { - continue; - } - seen.add(dedupeKey); + const dedupeKey = `${coords.line}:${coords.column}:${coords.match}`; + if (seen.has(dedupeKey)) { + continue; } + seen.add(dedupeKey); verified.push({ raw: candidate, coords }); } diff --git a/src/findings/scorer.ts b/src/findings/scorer.ts index 29f6be15..349a3b49 100644 --- a/src/findings/scorer.ts +++ b/src/findings/scorer.ts @@ -25,7 +25,7 @@ export function scoreFindings(params: { }): ScoredFindings { const scored = calculateScore( params.verifiedViolations, - params.wordCount, + Math.max(1, params.wordCount), { ...(params.strictness !== undefined ? { strictness: params.strictness } : {}), ...(params.promptSeverity !== undefined diff --git a/src/prompts/prompt-validator.ts b/src/prompts/prompt-validator.ts index 55897553..6275f731 100644 --- a/src/prompts/prompt-validator.ts +++ b/src/prompts/prompt-validator.ts @@ -1,5 +1,6 @@ import { PromptFile, PromptMeta, PromptCriterionSpec } from '../schemas/prompt-schemas'; import { Severity } from '../evaluators/types'; +import { DEFAULT_TARGET_FLAGS } from './target'; export type ValidationLevel = Severity; export interface Validation { @@ -55,7 +56,7 @@ export function validatePrompt(p: PromptFile): Validation[] { out.push({ file: p.filename, level: Severity.ERROR, message: 'Invalid global target.group' }); } if (meta.target.regex) { - try { new RegExp(meta.target.regex, meta.target.flags || ''); } catch { + try { new RegExp(meta.target.regex, meta.target.flags || DEFAULT_TARGET_FLAGS); } catch { out.push({ file: p.filename, level: Severity.ERROR, message: 'Invalid global target.regex' }); } } diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index a02972ff..a370f52c 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -175,22 +175,7 @@ export type ScoredEvaluation = { severity: typeof Severity.WARNING | typeof Severity.ERROR; message: string; reasoning?: string; - violations: Array<{ - line?: number; - analysis: string; - message?: string; - suggestion?: string; - fix?: string; - quoted_text?: string; - context_before?: string; - context_after?: string; - criterionName?: string; - description?: string; - rule_quote?: string; - checks?: GateChecks; - check_notes?: GateCheckNotes; - confidence?: number; - }>; + violations: Array; usage?: TokenUsage; raw_model_output?: unknown; }; @@ -202,4 +187,3 @@ export type PromptEvaluationResult = { usage?: TokenUsage; raw_model_output?: unknown; }; - diff --git a/src/prompts/target.ts b/src/prompts/target.ts index 9224b616..522d52c4 100644 --- a/src/prompts/target.ts +++ b/src/prompts/target.ts @@ -3,6 +3,8 @@ export type { TargetSpec } from '../schemas/prompt-schemas'; import type { TargetSpec } from '../schemas/prompt-schemas'; +export const DEFAULT_TARGET_FLAGS = 'mu'; + export function checkTarget( content: string, target?: TargetSpec, @@ -10,7 +12,7 @@ export function checkTarget( if (!target || !target.regex) return { missing: false }; let match: RegExpExecArray | null = null; try { - const flags = target.flags || 'mu'; + const flags = target.flags || DEFAULT_TARGET_FLAGS; const re = new RegExp(target.regex, flags); match = re.exec(content); } catch { diff --git a/tests/findings/processor.test.ts b/tests/findings/processor.test.ts index 6b09e971..f5ae7be9 100644 --- a/tests/findings/processor.test.ts +++ b/tests/findings/processor.test.ts @@ -167,12 +167,19 @@ describe('processFindings', () => { expect(result.hadOperationalErrors).toBe(false); }); - it('deduplicates verified findings by quoted_text and line', () => { + it('deduplicates candidates that resolve to the same verified anchor', () => { const result = processFindings( - baseInput([ - violation({ quoted_text: 'Alpha text', line: 1 }), - violation({ quoted_text: 'Alpha text', line: 1, message: 'dup' }), - ]), + baseInput( + [ + violation({ quoted_text: 'Alpha text Beta', line: 1 }), + violation({ + quoted_text: 'Alpha text Beta extra', + line: 1, + message: 'duplicate anchor', + }), + ], + { targetContent: 'Alpha text Beta\n' }, + ), ); expect(result.findings).toHaveLength(1); diff --git a/tests/findings/scorer.test.ts b/tests/findings/scorer.test.ts index c7c7d19b..8e660bf2 100644 --- a/tests/findings/scorer.test.ts +++ b/tests/findings/scorer.test.ts @@ -86,4 +86,15 @@ describe('scoreFindings', () => { const direct = calculateScore(verified, 100, { strictness: 'strict' }); expect(result.score).toBe(direct.final_score); }); + + it('returns a perfect score for an empty zero-word target', () => { + const result = scoreFindings({ + verifiedViolations: [], + wordCount: 0, + }); + + expect(result.score).toBe(10); + expect(result.scoreText).toBe('10.0/10'); + expect(result.findingCount).toBe(0); + }); }); diff --git a/tests/scoring-types.test.ts b/tests/scoring-types.test.ts index 08ee7357..ba799007 100644 --- a/tests/scoring-types.test.ts +++ b/tests/scoring-types.test.ts @@ -5,6 +5,24 @@ import type { PromptFile } from "../src/schemas/prompt-schemas"; import type { EvaluationLLMResult } from "../src/prompts/schema"; import type { SearchProvider } from "../src/providers/search-provider"; +const FULLY_SUPPORTED_CHECKS = { + rule_supports_claim: true, + evidence_exact: true, + context_supports_violation: true, + plausible_non_violation: false, + fix_is_drop_in: true, + fix_preserves_meaning: true, +}; + +const CHECK_NOTES = { + rule_supports_claim: "Supported", + evidence_exact: "Exact", + context_supports_violation: "Supported", + plausible_non_violation: "None", + fix_is_drop_in: "Drop-in", + fix_preserves_meaning: "Preserved", +}; + describe("Scoring Types", () => { const mockLlmProvider = { runPromptStructured: vi.fn(), @@ -29,22 +47,37 @@ describe("Scoring Types", () => { // Mock LLM returning violations only wrapped in LLMResult const mockLlmResponse: LLMResult = { data: { + reasoning: "Two issues found", violations: [ { + line: 1, description: "Issue 1", analysis: "First issue found", + message: "First issue", suggestion: "", + fix: "", quoted_text: "", context_before: "", context_after: "", + rule_quote: "", + checks: FULLY_SUPPORTED_CHECKS, + check_notes: CHECK_NOTES, + confidence: 0.9, }, { + line: 2, description: "Issue 2", analysis: "Second issue found", + message: "Second issue", suggestion: "", + fix: "", quoted_text: "", context_before: "", context_after: "", + rule_quote: "", + checks: FULLY_SUPPORTED_CHECKS, + check_notes: CHECK_NOTES, + confidence: 0.9, }, ], }, @@ -66,6 +99,7 @@ describe("Scoring Types", () => { const mockLlmResponse: LLMResult = { data: { + reasoning: "No issues found", violations: [], }, }; diff --git a/tests/validator.test.ts b/tests/validator.test.ts index c695bfed..a94aae9d 100644 --- a/tests/validator.test.ts +++ b/tests/validator.test.ts @@ -52,4 +52,25 @@ describe('PromptValidator', () => { expect(res.errors.some(e => /Invalid regex flags/i.test(e.message))).toBe(true); expect(res.errors.some(e => /Invalid global target\.regex/i.test(e.message))).toBe(true); }); + + it('validates regex with the default execution flags', () => { + const root = mkdtempSync(path.join(tmpdir(), 'vlint-')); + const promptsDir = path.join(root, 'prompts'); + mkdirSync(promptsDir, { recursive: true }); + const yaml = [ + 'id: TestPrompt', + 'name: Test Prompt', + 'target:', + " regex: '\\a'", + 'criteria:', + ' - name: A', + ' id: A', + ].join('\n'); + writePrompt(promptsDir, 'bad-default-flags.md', yaml); + + const { prompts } = loadRules(promptsDir); + const res = validateAll(prompts); + + expect(res.errors.some(e => /Invalid global target\.regex/i.test(e.message))).toBe(true); + }); });