diff --git a/.gitignore b/.gitignore index bff2607..0bf7816 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ coverage/ .DS_Store .next/ out/ +test-results/ +playwright-report/ diff --git a/packages/reporter/examples/README.md b/packages/reporter/examples/README.md new file mode 100644 index 0000000..7d18f09 --- /dev/null +++ b/packages/reporter/examples/README.md @@ -0,0 +1,21 @@ +# Example: wiring the Flakemetry reporter + +```ts +// playwright.config.ts +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + reporter: [['@flakemetry/playwright-reporter']], +}) +``` + +Configure via environment variables: + +```bash +FLAKEMETRY_ENDPOINT=https://ingest.example.com \ +FLAKEMETRY_TOKEN=fmk_xxx \ +FLAKEMETRY_PROJECT=acme/web \ + npx playwright test +``` + +Off CI or without a token the reporter is a no-op on delivery (fail-open). Set `FLAKEMETRY_OUTPUT_FILE=batch.json` to write the ingest batch to disk for inspection. diff --git a/packages/reporter/examples/playwright.config.ts b/packages/reporter/examples/playwright.config.ts new file mode 100644 index 0000000..a7dc139 --- /dev/null +++ b/packages/reporter/examples/playwright.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + reporter: [['@flakemetry/playwright-reporter']], +}) diff --git a/packages/reporter/package.json b/packages/reporter/package.json index c54d57d..036b406 100644 --- a/packages/reporter/package.json +++ b/packages/reporter/package.json @@ -32,11 +32,22 @@ "build": "tsup", "typecheck": "tsc -p tsconfig.json", "lint": "eslint .", - "clean": "rm -rf dist .turbo" + "clean": "rm -rf dist .turbo", + "test": "vitest run" }, "devDependencies": { "@flakemetry/eslint-config": "workspace:*", "@flakemetry/tsconfig": "workspace:*", - "@flakemetry/tsup-config": "workspace:*" + "@flakemetry/tsup-config": "workspace:*", + "@playwright/test": "^1.49.1", + "@types/node": "^22.10.5", + "vitest": "^3.2.4" + }, + "dependencies": { + "@flakemetry/contracts": "workspace:*", + "@flakemetry/sdk": "workspace:*" + }, + "peerDependencies": { + "@playwright/test": ">=1.44" } } diff --git a/packages/reporter/src/__tests__/mapping.test.ts b/packages/reporter/src/__tests__/mapping.test.ts new file mode 100644 index 0000000..4b11ec4 --- /dev/null +++ b/packages/reporter/src/__tests__/mapping.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest' + +import { buildIdempotencyKey, deriveSuite, resolveRunContext, statusFromResult } from '../mapping' + +describe('statusFromResult', () => { + it('maps a first-attempt pass to pass and a retried pass to flaky', () => { + expect(statusFromResult('passed', 0)).toBe('pass') + expect(statusFromResult('passed', 1)).toBe('flaky') + }) + + it('maps every failure kind to fail', () => { + expect(statusFromResult('failed', 0)).toBe('fail') + expect(statusFromResult('timedOut', 0)).toBe('fail') + expect(statusFromResult('interrupted', 0)).toBe('fail') + }) + + it('maps skipped to skip', () => { + expect(statusFromResult('skipped', 0)).toBe('skip') + }) +}) + +describe('deriveSuite', () => { + it('joins only describe titles, ignoring root/project/file nodes', () => { + const suite = deriveSuite([ + { type: 'root', title: '' }, + { type: 'project', title: 'chromium' }, + { type: 'file', title: 'login.spec.ts' }, + { type: 'describe', title: 'auth' }, + { type: 'describe', title: 'login' }, + ]) + expect(suite).toBe('auth > login') + }) + + it('is empty when there are no describe blocks', () => { + expect(deriveSuite([{ type: 'file', title: 'a.spec.ts' }])).toBe('') + }) +}) + +describe('resolveRunContext', () => { + it('reads github actions context including pr number from the ref', () => { + const context = resolveRunContext({ + GITHUB_ACTIONS: 'true', + GITHUB_SHA: 'abc1234', + GITHUB_REF_NAME: 'feat/login', + GITHUB_RUN_ID: '9000001', + GITHUB_EVENT_NAME: 'pull_request', + GITHUB_REF: 'refs/pull/42/merge', + FLAKEMETRY_PROJECT: 'acme/web', + }) + expect(context.ciProvider).toBe('github_actions') + expect(context.trigger).toBe('pull_request') + expect(context.commitSha).toBe('abc1234') + expect(context.prNumber).toBe(42) + expect(context.project).toBe('acme/web') + }) + + it('falls back to local defaults off CI', () => { + const context = resolveRunContext({}) + expect(context.ciProvider).toBe('local') + expect(context.trigger).toBe('manual') + expect(context.prNumber).toBeNull() + }) + + it('treats empty-string env vars as absent', () => { + const context = resolveRunContext({ + GITHUB_SHA: '', + GITHUB_REF_NAME: '', + FLAKEMETRY_COMMIT_SHA: 'deadbeef', + }) + expect(context.commitSha).toBe('deadbeef') + expect(context.branch).toBe('local') + }) +}) + +describe('buildIdempotencyKey', () => { + const base = { + project: 'acme/web', + commitSha: 'abc', + branch: 'main', + ciProvider: 'github_actions' as const, + trigger: 'push' as const, + ciRunId: '9000001', + prNumber: null, + } + + it('derives a stable key from the ci run and attempt', () => { + expect(buildIdempotencyKey(base, { GITHUB_RUN_ATTEMPT: '2' })).toBe('github_actions-9000001-2') + }) + + it('honors an explicit override', () => { + expect(buildIdempotencyKey(base, { FLAKEMETRY_IDEMPOTENCY_KEY: 'custom-key-1234' })).toBe( + 'custom-key-1234', + ) + }) + + it('generates a local key when there is no ci run id', () => { + const key = buildIdempotencyKey({ ...base, ciRunId: null }, {}) + expect(key.startsWith('local-')).toBe(true) + expect(key.length).toBeGreaterThanOrEqual(8) + }) +}) diff --git a/packages/reporter/src/__tests__/reporter.test.ts b/packages/reporter/src/__tests__/reporter.test.ts new file mode 100644 index 0000000..3184d76 --- /dev/null +++ b/packages/reporter/src/__tests__/reporter.test.ts @@ -0,0 +1,121 @@ +import { readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { ingestRunBatchSchema } from '@flakemetry/contracts' +import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '@playwright/test/reporter' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import FlakemetryReporter from '../reporter' + +const rootDir = '/repo' + +const makeSuiteChain = (describeTitle: string): Suite => { + const root = { type: 'root', title: '', parent: undefined } as unknown as Suite + const file = { type: 'file', title: 'login.spec.ts', parent: root } as unknown as Suite + return { type: 'describe', title: describeTitle, parent: file } as unknown as Suite +} + +const makeTest = (id: string, title: string, describeTitle: string): TestCase => + ({ + id, + title, + location: { file: `${rootDir}/e2e/login.spec.ts`, line: 1, column: 1 }, + parent: makeSuiteChain(describeTitle), + }) as unknown as TestCase + +const makeResult = ( + status: TestResult['status'], + retry: number, + error?: TestResult['error'], +): TestResult => + ({ + status, + retry, + duration: 1500, + startTime: new Date('2026-07-16T10:00:00Z'), + error, + }) as unknown as TestResult + +const drive = async (outputFile: string) => { + const reporter = new FlakemetryReporter({ outputFile }) + reporter.onBegin({ rootDir } as FullConfig, {} as Suite) + + const pass = makeTest('t1', 'logs in', 'auth') + reporter.onTestEnd(pass, makeResult('passed', 0)) + + const fail = makeTest('t2', 'rejects invalid', 'auth') + reporter.onTestEnd( + fail, + makeResult('failed', 0, { + message: 'expected true', + stack: 'at login:5', + } as TestResult['error']), + ) + + const flaky = makeTest('t3', 'completes payment', 'checkout') + reporter.onTestEnd( + flaky, + makeResult('failed', 0, { message: 'race', stack: 'at pay:9' } as TestResult['error']), + ) + reporter.onTestEnd(flaky, makeResult('passed', 1)) + + await reporter.onEnd({ status: 'failed' } as FullResult) +} + +describe('FlakemetryReporter lifecycle', () => { + beforeEach(() => { + for (const key of [ + 'GITHUB_ACTIONS', + 'GITHUB_SHA', + 'GITHUB_REF', + 'GITHUB_REF_NAME', + 'GITHUB_RUN_ID', + 'GITHUB_RUN_ATTEMPT', + 'GITHUB_EVENT_NAME', + 'FLAKEMETRY_ENDPOINT', + 'FLAKEMETRY_TOKEN', + 'FLAKEMETRY_IDEMPOTENCY_KEY', + ]) { + vi.stubEnv(key, '') + } + vi.stubEnv('FLAKEMETRY_PROJECT', 'acme/web') + vi.stubEnv('FLAKEMETRY_COMMIT_SHA', 'deadbeef') + }) + + afterEach(() => { + vi.unstubAllEnvs() + }) + + it('produces a contract-valid batch from a full run', async () => { + const outputFile = join(tmpdir(), `flakemetry-reporter-${process.hrtime.bigint()}.json`) + await drive(outputFile) + + const raw = JSON.parse(readFileSync(outputFile, 'utf8')) + const batch = ingestRunBatchSchema.parse(raw) + + expect(batch.resource.commitSha).toBe('deadbeef') + expect(batch.resource.ciProvider).toBe('local') + expect(batch.run.status).toBe('failed') + expect(batch.executions).toHaveLength(4) + }) + + it('maps statuses and links the flaky retry to its first attempt', async () => { + const outputFile = join(tmpdir(), `flakemetry-reporter-${process.hrtime.bigint()}.json`) + await drive(outputFile) + + const batch = ingestRunBatchSchema.parse(JSON.parse(readFileSync(outputFile, 'utf8'))) + const [pass, fail, flakyFirst, flakyRetry] = batch.executions + + expect(pass?.status).toBe('pass') + expect(fail?.status).toBe('fail') + expect(fail?.error?.message).toBe('expected true') + expect(flakyFirst?.status).toBe('fail') + expect(flakyFirst?.attempt).toBe(1) + expect(flakyRetry?.status).toBe('flaky') + expect(flakyRetry?.attempt).toBe(2) + expect(flakyRetry?.retryOfIndex).toBe(2) + expect(batch.executions[1]?.suite).toBe('auth') + expect(flakyRetry?.suite).toBe('checkout') + }) +}) diff --git a/packages/reporter/src/index.ts b/packages/reporter/src/index.ts index 49f298a..3fa544d 100644 --- a/packages/reporter/src/index.ts +++ b/packages/reporter/src/index.ts @@ -1,6 +1,3 @@ -export const PACKAGE = '@flakemetry/playwright-reporter' as const - -export const reporter = { - name: PACKAGE, - version: '0.0.0', -} as const +export * from './mapping' +export type { FlakemetryReporterOptions } from './reporter' +export { default } from './reporter' diff --git a/packages/reporter/src/mapping.ts b/packages/reporter/src/mapping.ts new file mode 100644 index 0000000..dfa5a2c --- /dev/null +++ b/packages/reporter/src/mapping.ts @@ -0,0 +1,65 @@ +import { randomUUID } from 'node:crypto' + +import type { CiProvider, RunTrigger, TestStatus } from '@flakemetry/contracts' +import type { RunContext } from '@flakemetry/sdk' + +export type PlaywrightStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted' + +export const statusFromResult = (status: PlaywrightStatus, retry: number): TestStatus => { + if (status === 'skipped') return 'skip' + if (status === 'passed') return retry > 0 ? 'flaky' : 'pass' + return 'fail' +} + +export interface SuiteNode { + type: string + title: string +} + +export const deriveSuite = (ancestors: readonly SuiteNode[]): string => + ancestors + .filter((node) => node.type === 'describe' && node.title.length > 0) + .map((node) => node.title) + .join(' > ') + +const prNumberFromRef = (ref: string | undefined): number | null => { + if (!ref) return null + const match = /refs\/pull\/(\d+)\//.exec(ref) + return match ? Number(match[1]) : null +} + +const pick = (value: string | undefined): string | undefined => + value && value.length > 0 ? value : undefined + +export const resolveRunContext = (env: Record): RunContext => { + const onGithub = env.GITHUB_ACTIONS === 'true' + const ciProvider: CiProvider = onGithub ? 'github_actions' : 'local' + const trigger: RunTrigger = onGithub + ? env.GITHUB_EVENT_NAME === 'pull_request' + ? 'pull_request' + : env.GITHUB_EVENT_NAME === 'schedule' + ? 'schedule' + : 'push' + : 'manual' + + return { + project: pick(env.FLAKEMETRY_PROJECT) ?? 'local/project', + commitSha: pick(env.GITHUB_SHA) ?? pick(env.FLAKEMETRY_COMMIT_SHA) ?? '0000000', + branch: pick(env.GITHUB_REF_NAME) ?? pick(env.FLAKEMETRY_BRANCH) ?? 'local', + ciProvider, + trigger, + ciRunId: pick(env.GITHUB_RUN_ID) ?? null, + prNumber: prNumberFromRef(pick(env.GITHUB_REF)), + } +} + +export const buildIdempotencyKey = ( + context: RunContext, + env: Record, +): string => { + const explicit = env.FLAKEMETRY_IDEMPOTENCY_KEY + if (explicit) return explicit + if (context.ciRunId) + return `${context.ciProvider}-${context.ciRunId}-${env.GITHUB_RUN_ATTEMPT ?? '1'}` + return `local-${randomUUID()}` +} diff --git a/packages/reporter/src/reporter.ts b/packages/reporter/src/reporter.ts new file mode 100644 index 0000000..7ed78ce --- /dev/null +++ b/packages/reporter/src/reporter.ts @@ -0,0 +1,116 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, relative } from 'node:path' + +import type { IngestRunBatch } from '@flakemetry/contracts' +import { IngestClient, type RunContext, TestRunRecorder } from '@flakemetry/sdk' +import type { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestResult, +} from '@playwright/test/reporter' + +import { + buildIdempotencyKey, + deriveSuite, + type PlaywrightStatus, + resolveRunContext, + statusFromResult, + type SuiteNode, +} from './mapping' + +export interface FlakemetryReporterOptions { + endpoint?: string + token?: string + outputFile?: string +} + +const collectAncestors = (test: TestCase): SuiteNode[] => { + const ancestors: SuiteNode[] = [] + let current: Suite | undefined = test.parent + while (current) { + ancestors.unshift({ type: current.type, title: current.title }) + current = current.parent + } + return ancestors +} + +export default class FlakemetryReporter implements Reporter { + private readonly options: FlakemetryReporterOptions + private readonly env: Record + private recorder: TestRunRecorder | null = null + private context: RunContext | null = null + private rootDir = process.cwd() + private readonly firstAttemptIndex = new Map() + + constructor(options: FlakemetryReporterOptions = {}) { + this.options = options + this.env = process.env + } + + onBegin(config: FullConfig, _suite: Suite): void { + this.rootDir = config.rootDir + this.context = resolveRunContext(this.env) + this.recorder = new TestRunRecorder(this.context) + this.recorder.startRun(new Date()) + } + + onTestEnd(test: TestCase, result: TestResult): void { + if (!this.recorder) return + const index = this.recorder.recorded.length + const attempt = result.retry + 1 + const retryOfIndex = attempt > 1 ? (this.firstAttemptIndex.get(test.id) ?? null) : null + const error = result.error + ? { + type: result.error.value ?? undefined, + message: result.error.message ?? 'unknown error', + stack: result.error.stack ?? undefined, + } + : null + + this.recorder.record({ + filePath: relative(this.rootDir, test.location.file), + suite: deriveSuite(collectAncestors(test)), + title: test.title, + status: statusFromResult(result.status as PlaywrightStatus, result.retry), + attempt, + retryOfIndex, + startedAt: result.startTime, + durationMs: Math.round(result.duration), + error, + }) + + if (attempt === 1) this.firstAttemptIndex.set(test.id, index) + } + + async onEnd(result: FullResult): Promise { + if (!this.recorder || !this.context) return + this.recorder.finishRun(result.status === 'passed' ? 'passed' : 'failed', new Date()) + const batch = this.recorder.toIngestBatch(buildIdempotencyKey(this.context, this.env)) + + this.writeOutput(batch) + await this.deliver(batch) + } + + private writeOutput(batch: IngestRunBatch): void { + const outputFile = this.options.outputFile ?? this.env.FLAKEMETRY_OUTPUT_FILE + if (!outputFile) return + mkdirSync(dirname(outputFile), { recursive: true }) + writeFileSync(outputFile, JSON.stringify(batch, null, 2)) + } + + private async deliver(batch: IngestRunBatch): Promise { + const endpoint = this.options.endpoint ?? this.env.FLAKEMETRY_ENDPOINT + const token = this.options.token ?? this.env.FLAKEMETRY_TOKEN + if (!endpoint || !token) return + const client = new IngestClient({ endpoint, token }) + const outcome = await client.send(batch) + if (!outcome.ok) { + process.stderr.write( + `flakemetry: upload skipped (${outcome.error ?? `status ${outcome.status}`})\n`, + ) + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 33d5c98..c65cd79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -183,6 +183,13 @@ importers: version: 8.64.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) packages/reporter: + dependencies: + '@flakemetry/contracts': + specifier: workspace:* + version: link:../contracts + '@flakemetry/sdk': + specifier: workspace:* + version: link:../sdk devDependencies: '@flakemetry/eslint-config': specifier: workspace:* @@ -193,6 +200,15 @@ importers: '@flakemetry/tsup-config': specifier: workspace:* version: link:../tsup-config + '@playwright/test': + specifier: ^1.49.1 + version: 1.61.1 + '@types/node': + specifier: ^22.10.5 + version: 22.20.1 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0) packages/sdk: dependencies: @@ -742,6 +758,11 @@ packages: resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} engines: {node: '>=14'} + '@playwright/test@1.61.1': + resolution: {integrity: sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==} + engines: {node: '>=18'} + hasBin: true + '@prisma/client@6.19.3': resolution: {integrity: sha512-mKq3jQFhjvko5LTJFHGilsuQs+W+T3Gm451NzuTDGQxwCzwXHYnIu2zGkRoW+Exq3Rob7yp2MfzSrdIiZVhrBg==} engines: {node: '>=18.18'} @@ -1383,6 +1404,11 @@ packages: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1671,6 +1697,16 @@ packages: pkg-types@2.3.1: resolution: {integrity: sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg==} + playwright-core@1.61.1: + resolution: {integrity: sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.1: + resolution: {integrity: sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==} + engines: {node: '>=18'} + hasBin: true + postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -2486,6 +2522,10 @@ snapshots: '@opentelemetry/semantic-conventions@1.43.0': {} + '@playwright/test@1.61.1': + dependencies: + playwright: 1.61.1 + '@prisma/client@6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3)': optionalDependencies: prisma: 6.19.3(typescript@5.9.3) @@ -3150,6 +3190,9 @@ snapshots: jsonfile: 4.0.0 universalify: 0.1.2 + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -3401,6 +3444,14 @@ snapshots: exsolve: 1.1.0 pathe: 2.0.3 + playwright-core@1.61.1: {} + + playwright@1.61.1: + dependencies: + playwright-core: 1.61.1 + optionalDependencies: + fsevents: 2.3.2 + postcss-load-config@6.0.1(jiti@2.7.0)(postcss@8.5.19)(tsx@4.23.1)(yaml@2.9.0): dependencies: lilconfig: 3.1.3