feat: AI-driven PR comment resolution

.github/workflows/feature.yml

@@ -82,3 +82,32 @@ jobs:
 
       - name: Test
         run: pnpm test
+
+  resolve-pr-comments:
+    name: resolve-pr-comments
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [typecheck, lint, format-check, test]
+    if: >-
+      github.event.action != 'opened' &&
+      !github.event.pull_request.head.repo.fork
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup
+        uses: ./.github/actions/setup
+
+      - name: Resolve PR comments
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          pnpm run:resolve-pr-comments -- \
+            --pr=${{ github.event.pull_request.number }} \
+            --repo=${{ github.repository }}

README.md

@@ -1,6 +1,6 @@
 # cli-agent-sandbox
 
-A minimal TypeScript CLI sandbox for testing agent workflows and safe web scraping. This is a single-package repo built with [`@openai/agents`](https://github.com/openai/openai-agents-js), and it includes a guestbook demo, a Finnish name explorer CLI, a publication scraping pipeline with a Playwright-based scraper for JS-rendered pages, an ETF backtest CLI, an agent evals CLI, an AI usage summary CLI, a PR comment fixer CLI, an update-docs CLI for diff-driven doc sync, and agent tools scoped to `tmp` with strong safety checks.
+A minimal TypeScript CLI sandbox for testing agent workflows and safe web scraping. This is a single-package repo built with [`@openai/agents`](https://github.com/openai/openai-agents-js), and it includes a guestbook demo, a Finnish name explorer CLI, a publication scraping pipeline with a Playwright-based scraper for JS-rendered pages, an ETF backtest CLI, an agent evals CLI, an AI usage summary CLI, a PR comment fixer CLI, a PR comment resolver CLI, an update-docs CLI for diff-driven doc sync, and agent tools scoped to `tmp` with strong safety checks.
 
 ## Quick Start
 
@@ -15,7 +15,8 @@ A minimal TypeScript CLI sandbox for testing agent workflows and safe web scrapi
 9. (Optional) Run ETF backtest: `pnpm run:etf-backtest -- --isin=IE00B5BMR087` (requires Python setup below)
 10. (Optional) Summarize AI usage: `pnpm ai:usage --since 7d`
 11. (Optional) Fetch PR comments: `pnpm run:fix-pr-comments -- --pr=10`
-12. (Optional) Update docs from a branch diff: `pnpm run:update-docs`
+12. (Optional) Resolve addressed PR comments: `pnpm run:resolve-pr-comments -- --pr=10`
+13. (Optional) Update docs from a branch diff: `pnpm run:update-docs`
 
 ### Python Setup (for ETF backtest)
 
@@ -38,6 +39,7 @@ pip install numpy pandas torch
 | `pnpm run:scrape-publications` | Scrape publication links and build a review page             |
 | `pnpm run:etf-backtest`        | Run ETF backtest + feature optimizer (requires Python)       |
 | `pnpm run:fix-pr-comments`     | Fetch PR comments, write markdown/JSON, optionally run Codex |
+| `pnpm run:resolve-pr-comments` | Analyze PR comments vs diff, reply + resolve addressed ones  |
 | `pnpm run:update-docs`         | Generate a branch diff and optionally run Codex to sync docs |
 | `pnpm ai:usage`                | Summarize Claude/Codex token usage for a repo                |
 | `pnpm typecheck`               | Run TypeScript type checking                                 |
@@ -143,6 +145,25 @@ Notes:
 - Review comments marked `fixed` in `answers.json` are skipped in later runs (and `review-comments.json` only includes unfixed entries).
 - Runs `pnpm typecheck`, `pnpm lint`, and `pnpm format` at the end.
 
+## Resolve PR comments
+
+The `run:resolve-pr-comments` CLI analyzes inline review comments against the current diff, then replies to and resolves any comments already addressed.
+
+Usage:
+
+```
+pnpm run:resolve-pr-comments -- --pr=10
+pnpm run:resolve-pr-comments -- --pr=10 --repo=owner/repo
+pnpm run:resolve-pr-comments -- --pr=10 --base=main
+pnpm run:resolve-pr-comments -- --pr=10 --dry-run
+```
+
+Notes:
+
+- Requires the `gh` CLI and authentication (`gh auth login`).
+- Writes analysis JSON to `tmp/resolve-pr-comments/pr-<number>/analysis.json`.
+- Use `--dry-run` to avoid posting replies/resolutions.
+
 ## Update docs
 
 The `run:update-docs` CLI compares the current branch against a base branch, writes a diff summary, and optionally launches Codex to sync documentation.
@@ -218,6 +239,13 @@ src/
 │   │   ├── types/             # CLI schemas
 │   │   │   └── schemas.ts     # CLI args + comment schemas
 │   │   └── clients/           # GitHub + formatting pipeline
+│   ├── resolve-pr-comments/
+│   │   ├── main.ts            # Resolve PR comments CLI entry point
+│   │   ├── README.md          # Resolve PR comments CLI docs
+│   │   ├── constants.ts       # CLI constants
+│   │   ├── types/             # CLI schemas
+│   │   │   └── schemas.ts     # CLI args + analysis schemas
+│   │   └── clients/           # GitHub + analysis pipeline
 │   ├── update-docs/
 │   │   ├── main.ts            # Update docs CLI entry point
 │   │   ├── README.md          # Update docs CLI docs

package.json

@@ -13,6 +13,7 @@
     "scaffold:cli": "tsx scripts/scaffold-cli.ts",
     "run:fix-pr-comments": "tsx src/cli/fix-pr-comments/main.ts",
     "run:update-docs": "tsx src/cli/update-docs/main.ts",
+    "run:resolve-pr-comments": "tsx src/cli/resolve-pr-comments/main.ts",
     "node:tsx": "node --disable-warning=ExperimentalWarning --import tsx",
     "typecheck": "tsc --noEmit",
     "lint": "eslint .",

src/cli/fix-pr-comments/clients/comment-formatter.test.ts

@@ -18,6 +18,7 @@ describe("CommentFormatter", () => {
     const reviewComment: ReviewComment = {
       ...baseComment,
       id: 1,
+      node_id: "PRRC_1",
       path: "src/index.ts",
       line: 42,
       original_line: null,
@@ -38,6 +39,7 @@ describe("CommentFormatter", () => {
     const reviewComment: ReviewComment = {
       ...baseComment,
       id: 2,
+      node_id: "PRRC_2",
       path: "src/index.ts",
       line: null,
       original_line: null,

src/cli/resolve-pr-comments/README.md

@@ -0,0 +1,63 @@
+# Resolve PR Comments CLI
+
+Analyze review comments on a GitHub PR against the current diff, then reply to addressed or uncertain comments and resolve those marked addressed.
+
+## Run
+
+```bash
+pnpm run:resolve-pr-comments -- --pr=10
+pnpm run:resolve-pr-comments -- --pr=10 --repo=owner/repo
+pnpm run:resolve-pr-comments -- --pr=10 --base=main
+pnpm run:resolve-pr-comments -- --pr=10 --dry-run
+```
+
+## Arguments
+
+- `--pr` (required): PR number to analyze.
+- `--repo` (optional): Repository in `owner/repo` format. Defaults to `gh repo view`.
+- `--base` (default: `main`): Base ref for `git diff <base>...HEAD`.
+- `--dry-run` (default: false): Log replies/resolutions without posting.
+
+## Workflow
+
+1. Ensures `gh` is installed and authenticated
+2. Resolves repo (`--repo` override or `gh repo view`)
+3. Fetches inline review comments for the PR
+4. Gets the git diff for `base...HEAD`
+5. Analyzes comments vs diff with a single AI request
+6. Writes analysis JSON under `tmp/resolve-pr-comments/pr-<number>/`
+7. Replies to addressed or uncertain comments and resolves addressed ones (skipped in dry-run)
+
+## Output
+
+Writes analysis to:
+
+```
+tmp/resolve-pr-comments/pr-<number>/analysis.json
+```
+
+## Prerequisites
+
+- `gh` CLI installed and authenticated (`gh auth login`)
+- `OPENAI_API_KEY` set for the analysis model
+- Local git checkout with the base ref available (e.g., `main`)
+
+## Flowchart
+
+```mermaid
+flowchart TD
+  A["Start"] --> B["Parse args"]
+  B --> C["Check gh auth + resolve repo"]
+  C --> D["Fetch review comments"]
+  D --> E["Get git diff base...HEAD"]
+  E --> F["AI analyze comments vs diff"]
+  F --> G["Write analysis.json"]
+  G --> H["Reply + resolve addressed comments"]
+  H --> I["Done"]
+```
+
+## Internals
+
+- `ResolvePrPipeline` orchestrates GitHub fetch, diff capture, analysis, and resolution
+- `CommentAnalyzer` uses `AgentRunner` to classify comments as addressed or not
+- `CommentResolver` posts replies and resolves threads via `GitHubClient`

src/cli/resolve-pr-comments/clients/comment-analyzer.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+
+import { truncateDiff } from "./comment-analyzer";
+
+describe("truncateDiff", () => {
+  it("returns original diff when within limit", () => {
+    const result = truncateDiff("abc", 10);
+
+    expect(result.diff).toBe("abc");
+    expect(result.truncated).toBe(false);
+    expect(result.originalLength).toBe(3);
+  });
+
+  it("truncates and annotates diff when over limit", () => {
+    const result = truncateDiff("abcdef", 3);
+
+    expect(result.diff).toContain("abc");
+    expect(result.diff).toContain("diff truncated to 3 characters");
+    expect(result.truncated).toBe(true);
+    expect(result.originalLength).toBe(6);
+  });
+});

src/cli/resolve-pr-comments/clients/comment-analyzer.ts

@@ -0,0 +1,84 @@
+import { AgentRunner } from "~clients/agent-runner";
+import type { ReviewComment } from "~clients/github-client";
+import type { Logger } from "~clients/logger";
+
+import { ANALYSIS_PROMPT_TEMPLATE, MAX_DIFF_CHARS } from "../constants";
+import type { AnalysisResult } from "../types/schemas";
+import { AnalysisResultSchema } from "../types/schemas";
+
+type CommentAnalyzerOptions = {
+  logger: Logger;
+};
+
+type AnalyzeOptions = {
+  comments: ReviewComment[];
+  diff: string;
+};
+
+export const truncateDiff = (
+  diff: string,
+  maxChars: number
+): { diff: string; truncated: boolean; originalLength: number } => {
+  if (diff.length <= maxChars) {
+    return { diff, truncated: false, originalLength: diff.length };
+  }
+
+  const truncatedDiff = diff.slice(0, maxChars);
+  return {
+    diff: `${truncatedDiff}\n\n... [diff truncated to ${maxChars} characters]\n`,
+    truncated: true,
+    originalLength: diff.length,
+  };
+};
+
+/**
+ * AI-powered comment analyzer that determines if comments are addressed by a diff.
+ * Makes a single API call to analyze all comments together.
+ */
+export class CommentAnalyzer {
+  private logger: Logger;
+
+  constructor(options: CommentAnalyzerOptions) {
+    this.logger = options.logger;
+  }
+
+  async analyze({ comments, diff }: AnalyzeOptions): Promise<AnalysisResult> {
+    const commentsJson = JSON.stringify(
+      comments.map((c) => ({
+        id: c.id,
+        path: c.path,
+        line: c.line ?? c.original_line,
+        body: c.body,
+        user: c.user.login,
+      })),
+      null,
+      2
+    );
+
+    const truncatedDiff = truncateDiff(diff, MAX_DIFF_CHARS);
+    const prompt = ANALYSIS_PROMPT_TEMPLATE(commentsJson, truncatedDiff.diff);
+
+    if (truncatedDiff.truncated) {
+      this.logger.warn("Diff truncated for analysis", {
+        originalLength: truncatedDiff.originalLength,
+        maxChars: MAX_DIFF_CHARS,
+      });
+    }
+
+    const runner = new AgentRunner<AnalysisResult>({
+      name: "comment-analyzer",
+      model: "gpt-5-mini",
+      tools: [],
+      outputType: AnalysisResultSchema,
+      instructions:
+        "Analyze PR comments to determine if they are addressed by the diff.",
+      logger: this.logger,
+      stateless: true,
+    });
+
+    this.logger.info("Analyzing comments with AI", { count: comments.length });
+    const result = await runner.run({ prompt });
+
+    return result.finalOutput as AnalysisResult;
+  }
+}

src/cli/resolve-pr-comments/clients/comment-resolver.ts

@@ -0,0 +1,96 @@
+import { GitHubClient } from "~clients/github-client";
+import type { Logger } from "~clients/logger";
+
+import type { CommentAnalysis, PrContext } from "../types/schemas";
+
+type CommentResolverOptions = {
+  logger: Logger;
+};
+
+type ResolveOptions = {
+  analysis: CommentAnalysis;
+  ctx: PrContext;
+  dryRun: boolean;
+};
+
+/**
+ * Handles GitHub API operations for replying to and reacting to PR comments.
+ * Uses 👍 reaction to mark addressed comments instead of resolving threads.
+ */
+export class CommentResolver {
+  private githubClient: GitHubClient;
+  private logger: Logger;
+
+  constructor(options: CommentResolverOptions) {
+    this.logger = options.logger;
+    this.githubClient = new GitHubClient({ logger: options.logger });
+  }
+
+  /**
+   * Get comment IDs that have already been marked with 👍 reaction.
+   */
+  async getAlreadyAddressedIds(
+    ctx: PrContext,
+    commentIds: number[]
+  ): Promise<Set<number>> {
+    return this.githubClient.getCommentIdsWithReaction(ctx, commentIds, "+1");
+  }
+
+  /**
+   * Get comment IDs that were previously marked as uncertain with 👀 reaction.
+   */
+  async getPreviouslyUncertainIds(
+    ctx: PrContext,
+    commentIds: number[]
+  ): Promise<Set<number>> {
+    return this.githubClient.getCommentIdsWithReaction(ctx, commentIds, "eyes");
+  }
+
+  async resolveComment({
+    analysis,
+    ctx,
+    dryRun,
+  }: ResolveOptions): Promise<boolean> {
+    if (analysis.status === "not_addressed") {
+      this.logger.debug("Skipping unaddressed comment", {
+        commentId: analysis.commentId,
+      });
+      return false;
+    }
+
+    const replyBody = analysis.suggestedReply;
+    const isAddressed = analysis.status === "addressed";
+
+    if (dryRun) {
+      this.logger.info(
+        isAddressed
+          ? "[DRY RUN] Would reply and react with 👍"
+          : "[DRY RUN] Would reply and react with 👀",
+        {
+          commentId: analysis.commentId,
+          status: analysis.status,
+          reply: replyBody,
+          reasoning: analysis.reasoning,
+        }
+      );
+      return false;
+    }
+
+    await this.githubClient.replyToComment(ctx, analysis.commentId, replyBody);
+    this.logger.info("Posted reply", {
+      commentId: analysis.commentId,
+      status: analysis.status,
+    });
+
+    if (isAddressed) {
+      await this.githubClient.reactToComment(ctx, analysis.commentId, "+1");
+      this.logger.info("Added 👍 reaction", { commentId: analysis.commentId });
+    } else {
+      // Uncertain - mark with 👀 so we don't re-process unless file changes
+      await this.githubClient.reactToComment(ctx, analysis.commentId, "eyes");
+      this.logger.info("Added 👀 reaction", { commentId: analysis.commentId });
+    }
+
+    return isAddressed;
+  }
+}