feat: add playwright reporter emitting ingest batches

.gitignore

@@ -9,3 +9,5 @@ coverage/
 .DS_Store
 .next/
 out/
+test-results/
+playwright-report/

packages/reporter/examples/README.md

@@ -0,0 +1,21 @@
+# Example: wiring the Flakemetry reporter
+
+```ts
+// playwright.config.ts
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  reporter: [['@flakemetry/playwright-reporter']],
+})
+```
+
+Configure via environment variables:
+
+```bash
+FLAKEMETRY_ENDPOINT=https://ingest.example.com \
+FLAKEMETRY_TOKEN=fmk_xxx \
+FLAKEMETRY_PROJECT=acme/web \
+  npx playwright test
+```
+
+Off CI or without a token the reporter is a no-op on delivery (fail-open). Set `FLAKEMETRY_OUTPUT_FILE=batch.json` to write the ingest batch to disk for inspection.

packages/reporter/examples/playwright.config.ts

@@ -0,0 +1,5 @@
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  reporter: [['@flakemetry/playwright-reporter']],
+})

packages/reporter/package.json

@@ -32,11 +32,22 @@
     "build": "tsup",
     "typecheck": "tsc -p tsconfig.json",
     "lint": "eslint .",
-    "clean": "rm -rf dist .turbo"
+    "clean": "rm -rf dist .turbo",
+    "test": "vitest run"
   },
   "devDependencies": {
     "@flakemetry/eslint-config": "workspace:*",
     "@flakemetry/tsconfig": "workspace:*",
-    "@flakemetry/tsup-config": "workspace:*"
+    "@flakemetry/tsup-config": "workspace:*",
+    "@playwright/test": "^1.49.1",
+    "@types/node": "^22.10.5",
+    "vitest": "^3.2.4"
+  },
+  "dependencies": {
+    "@flakemetry/contracts": "workspace:*",
+    "@flakemetry/sdk": "workspace:*"
+  },
+  "peerDependencies": {
+    "@playwright/test": ">=1.44"
   }
 }

packages/reporter/src/__tests__/mapping.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from 'vitest'
+
+import { buildIdempotencyKey, deriveSuite, resolveRunContext, statusFromResult } from '../mapping'
+
+describe('statusFromResult', () => {
+  it('maps a first-attempt pass to pass and a retried pass to flaky', () => {
+    expect(statusFromResult('passed', 0)).toBe('pass')
+    expect(statusFromResult('passed', 1)).toBe('flaky')
+  })
+
+  it('maps every failure kind to fail', () => {
+    expect(statusFromResult('failed', 0)).toBe('fail')
+    expect(statusFromResult('timedOut', 0)).toBe('fail')
+    expect(statusFromResult('interrupted', 0)).toBe('fail')
+  })
+
+  it('maps skipped to skip', () => {
+    expect(statusFromResult('skipped', 0)).toBe('skip')
+  })
+})
+
+describe('deriveSuite', () => {
+  it('joins only describe titles, ignoring root/project/file nodes', () => {
+    const suite = deriveSuite([
+      { type: 'root', title: '' },
+      { type: 'project', title: 'chromium' },
+      { type: 'file', title: 'login.spec.ts' },
+      { type: 'describe', title: 'auth' },
+      { type: 'describe', title: 'login' },
+    ])
+    expect(suite).toBe('auth > login')
+  })
+
+  it('is empty when there are no describe blocks', () => {
+    expect(deriveSuite([{ type: 'file', title: 'a.spec.ts' }])).toBe('')
+  })
+})
+
+describe('resolveRunContext', () => {
+  it('reads github actions context including pr number from the ref', () => {
+    const context = resolveRunContext({
+      GITHUB_ACTIONS: 'true',
+      GITHUB_SHA: 'abc1234',
+      GITHUB_REF_NAME: 'feat/login',
+      GITHUB_RUN_ID: '9000001',
+      GITHUB_EVENT_NAME: 'pull_request',
+      GITHUB_REF: 'refs/pull/42/merge',
+      FLAKEMETRY_PROJECT: 'acme/web',
+    })
+    expect(context.ciProvider).toBe('github_actions')
+    expect(context.trigger).toBe('pull_request')
+    expect(context.commitSha).toBe('abc1234')
+    expect(context.prNumber).toBe(42)
+    expect(context.project).toBe('acme/web')
+  })
+
+  it('falls back to local defaults off CI', () => {
+    const context = resolveRunContext({})
+    expect(context.ciProvider).toBe('local')
+    expect(context.trigger).toBe('manual')
+    expect(context.prNumber).toBeNull()
+  })
+
+  it('treats empty-string env vars as absent', () => {
+    const context = resolveRunContext({
+      GITHUB_SHA: '',
+      GITHUB_REF_NAME: '',
+      FLAKEMETRY_COMMIT_SHA: 'deadbeef',
+    })
+    expect(context.commitSha).toBe('deadbeef')
+    expect(context.branch).toBe('local')
+  })
+})
+
+describe('buildIdempotencyKey', () => {
+  const base = {
+    project: 'acme/web',
+    commitSha: 'abc',
+    branch: 'main',
+    ciProvider: 'github_actions' as const,
+    trigger: 'push' as const,
+    ciRunId: '9000001',
+    prNumber: null,
+  }
+
+  it('derives a stable key from the ci run and attempt', () => {
+    expect(buildIdempotencyKey(base, { GITHUB_RUN_ATTEMPT: '2' })).toBe('github_actions-9000001-2')
+  })
+
+  it('honors an explicit override', () => {
+    expect(buildIdempotencyKey(base, { FLAKEMETRY_IDEMPOTENCY_KEY: 'custom-key-1234' })).toBe(
+      'custom-key-1234',
+    )
+  })
+
+  it('generates a local key when there is no ci run id', () => {
+    const key = buildIdempotencyKey({ ...base, ciRunId: null }, {})
+    expect(key.startsWith('local-')).toBe(true)
+    expect(key.length).toBeGreaterThanOrEqual(8)
+  })
+})

packages/reporter/src/__tests__/reporter.test.ts

@@ -0,0 +1,121 @@
+import { readFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+import { ingestRunBatchSchema } from '@flakemetry/contracts'
+import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '@playwright/test/reporter'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import FlakemetryReporter from '../reporter'
+
+const rootDir = '/repo'
+
+const makeSuiteChain = (describeTitle: string): Suite => {
+  const root = { type: 'root', title: '', parent: undefined } as unknown as Suite
+  const file = { type: 'file', title: 'login.spec.ts', parent: root } as unknown as Suite
+  return { type: 'describe', title: describeTitle, parent: file } as unknown as Suite
+}
+
+const makeTest = (id: string, title: string, describeTitle: string): TestCase =>
+  ({
+    id,
+    title,
+    location: { file: `${rootDir}/e2e/login.spec.ts`, line: 1, column: 1 },
+    parent: makeSuiteChain(describeTitle),
+  }) as unknown as TestCase
+
+const makeResult = (
+  status: TestResult['status'],
+  retry: number,
+  error?: TestResult['error'],
+): TestResult =>
+  ({
+    status,
+    retry,
+    duration: 1500,
+    startTime: new Date('2026-07-16T10:00:00Z'),
+    error,
+  }) as unknown as TestResult
+
+const drive = async (outputFile: string) => {
+  const reporter = new FlakemetryReporter({ outputFile })
+  reporter.onBegin({ rootDir } as FullConfig, {} as Suite)
+
+  const pass = makeTest('t1', 'logs in', 'auth')
+  reporter.onTestEnd(pass, makeResult('passed', 0))
+
+  const fail = makeTest('t2', 'rejects invalid', 'auth')
+  reporter.onTestEnd(
+    fail,
+    makeResult('failed', 0, {
+      message: 'expected true',
+      stack: 'at login:5',
+    } as TestResult['error']),
+  )
+
+  const flaky = makeTest('t3', 'completes payment', 'checkout')
+  reporter.onTestEnd(
+    flaky,
+    makeResult('failed', 0, { message: 'race', stack: 'at pay:9' } as TestResult['error']),
+  )
+  reporter.onTestEnd(flaky, makeResult('passed', 1))
+
+  await reporter.onEnd({ status: 'failed' } as FullResult)
+}
+
+describe('FlakemetryReporter lifecycle', () => {
+  beforeEach(() => {
+    for (const key of [
+      'GITHUB_ACTIONS',
+      'GITHUB_SHA',
+      'GITHUB_REF',
+      'GITHUB_REF_NAME',
+      'GITHUB_RUN_ID',
+      'GITHUB_RUN_ATTEMPT',
+      'GITHUB_EVENT_NAME',
+      'FLAKEMETRY_ENDPOINT',
+      'FLAKEMETRY_TOKEN',
+      'FLAKEMETRY_IDEMPOTENCY_KEY',
+    ]) {
+      vi.stubEnv(key, '')
+    }
+    vi.stubEnv('FLAKEMETRY_PROJECT', 'acme/web')
+    vi.stubEnv('FLAKEMETRY_COMMIT_SHA', 'deadbeef')
+  })
+
+  afterEach(() => {
+    vi.unstubAllEnvs()
+  })
+
+  it('produces a contract-valid batch from a full run', async () => {
+    const outputFile = join(tmpdir(), `flakemetry-reporter-${process.hrtime.bigint()}.json`)
+    await drive(outputFile)
+
+    const raw = JSON.parse(readFileSync(outputFile, 'utf8'))
+    const batch = ingestRunBatchSchema.parse(raw)
+
+    expect(batch.resource.commitSha).toBe('deadbeef')
+    expect(batch.resource.ciProvider).toBe('local')
+    expect(batch.run.status).toBe('failed')
+    expect(batch.executions).toHaveLength(4)
+  })
+
+  it('maps statuses and links the flaky retry to its first attempt', async () => {
+    const outputFile = join(tmpdir(), `flakemetry-reporter-${process.hrtime.bigint()}.json`)
+    await drive(outputFile)
+
+    const batch = ingestRunBatchSchema.parse(JSON.parse(readFileSync(outputFile, 'utf8')))
+    const [pass, fail, flakyFirst, flakyRetry] = batch.executions
+
+    expect(pass?.status).toBe('pass')
+    expect(fail?.status).toBe('fail')
+    expect(fail?.error?.message).toBe('expected true')
+    expect(flakyFirst?.status).toBe('fail')
+    expect(flakyFirst?.attempt).toBe(1)
+    expect(flakyRetry?.status).toBe('flaky')
+    expect(flakyRetry?.attempt).toBe(2)
+    expect(flakyRetry?.retryOfIndex).toBe(2)
+    expect(batch.executions[1]?.suite).toBe('auth')
+    expect(flakyRetry?.suite).toBe('checkout')
+  })
+})

packages/reporter/src/index.ts

@@ -1,6 +1,3 @@
-export const PACKAGE = '@flakemetry/playwright-reporter' as const
-
-export const reporter = {
-  name: PACKAGE,
-  version: '0.0.0',
-} as const
+export * from './mapping'
+export type { FlakemetryReporterOptions } from './reporter'
+export { default } from './reporter'

packages/reporter/src/mapping.ts

@@ -0,0 +1,65 @@
+import { randomUUID } from 'node:crypto'
+
+import type { CiProvider, RunTrigger, TestStatus } from '@flakemetry/contracts'
+import type { RunContext } from '@flakemetry/sdk'
+
+export type PlaywrightStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'
+
+export const statusFromResult = (status: PlaywrightStatus, retry: number): TestStatus => {
+  if (status === 'skipped') return 'skip'
+  if (status === 'passed') return retry > 0 ? 'flaky' : 'pass'
+  return 'fail'
+}
+
+export interface SuiteNode {
+  type: string
+  title: string
+}
+
+export const deriveSuite = (ancestors: readonly SuiteNode[]): string =>
+  ancestors
+    .filter((node) => node.type === 'describe' && node.title.length > 0)
+    .map((node) => node.title)
+    .join(' > ')
+
+const prNumberFromRef = (ref: string | undefined): number | null => {
+  if (!ref) return null
+  const match = /refs\/pull\/(\d+)\//.exec(ref)
+  return match ? Number(match[1]) : null
+}
+
+const pick = (value: string | undefined): string | undefined =>
+  value && value.length > 0 ? value : undefined
+
+export const resolveRunContext = (env: Record<string, string | undefined>): RunContext => {
+  const onGithub = env.GITHUB_ACTIONS === 'true'
+  const ciProvider: CiProvider = onGithub ? 'github_actions' : 'local'
+  const trigger: RunTrigger = onGithub
+    ? env.GITHUB_EVENT_NAME === 'pull_request'
+      ? 'pull_request'
+      : env.GITHUB_EVENT_NAME === 'schedule'
+        ? 'schedule'
+        : 'push'
+    : 'manual'
+
+  return {
+    project: pick(env.FLAKEMETRY_PROJECT) ?? 'local/project',
+    commitSha: pick(env.GITHUB_SHA) ?? pick(env.FLAKEMETRY_COMMIT_SHA) ?? '0000000',
+    branch: pick(env.GITHUB_REF_NAME) ?? pick(env.FLAKEMETRY_BRANCH) ?? 'local',
+    ciProvider,
+    trigger,
+    ciRunId: pick(env.GITHUB_RUN_ID) ?? null,
+    prNumber: prNumberFromRef(pick(env.GITHUB_REF)),
+  }
+}
+
+export const buildIdempotencyKey = (
+  context: RunContext,
+  env: Record<string, string | undefined>,
+): string => {
+  const explicit = env.FLAKEMETRY_IDEMPOTENCY_KEY
+  if (explicit) return explicit
+  if (context.ciRunId)
+    return `${context.ciProvider}-${context.ciRunId}-${env.GITHUB_RUN_ATTEMPT ?? '1'}`
+  return `local-${randomUUID()}`
+}