From a3c877053318f0c843c41d936f6e5ed2165e74ca Mon Sep 17 00:00:00 2001 From: Andrii Kohut Date: Fri, 17 Jul 2026 12:41:30 +0200 Subject: [PATCH 1/2] feat: add playwright reporter emitting ingest batches Implement @flakemetry/playwright-reporter on top of the sdk: onBegin resolves the CI run context from the environment, onTestEnd records each attempt with status mapping (retried pass becomes flaky, timeouts and interruptions become fail) and links retries to their first attempt, and onEnd builds a contract-valid ingest batch and delivers it fail-open. Suite names come from describe blocks only, CI metadata and pr number are read from GitHub Actions env, and an output-file mode dumps the batch for inspection. Covered by pure mapping tests and a full-lifecycle integration test driving synthetic Playwright events. Closes #14. --- .gitignore | 2 + packages/reporter/examples/README.md | 21 ++++ .../reporter/examples/playwright.config.ts | 5 + packages/reporter/package.json | 15 ++- .../reporter/src/__tests__/mapping.test.ts | 91 ++++++++++++++ .../reporter/src/__tests__/reporter.test.ts | 110 +++++++++++++++++ packages/reporter/src/index.ts | 9 +- packages/reporter/src/mapping.ts | 62 ++++++++++ packages/reporter/src/reporter.ts | 116 ++++++++++++++++++ pnpm-lock.yaml | 51 ++++++++ 10 files changed, 474 insertions(+), 8 deletions(-) create mode 100644 packages/reporter/examples/README.md create mode 100644 packages/reporter/examples/playwright.config.ts create mode 100644 packages/reporter/src/__tests__/mapping.test.ts create mode 100644 packages/reporter/src/__tests__/reporter.test.ts create mode 100644 packages/reporter/src/mapping.ts create mode 100644 packages/reporter/src/reporter.ts diff --git a/.gitignore b/.gitignore index bff2607..0bf7816 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ coverage/ .DS_Store .next/ out/ +test-results/ +playwright-report/ diff --git a/packages/reporter/examples/README.md b/packages/reporter/examples/README.md new file mode 100644 index 0000000..7d18f09 --- /dev/null +++ b/packages/reporter/examples/README.md @@ -0,0 +1,21 @@ +# Example: wiring the Flakemetry reporter + +```ts +// playwright.config.ts +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + reporter: [['@flakemetry/playwright-reporter']], +}) +``` + +Configure via environment variables: + +```bash +FLAKEMETRY_ENDPOINT=https://ingest.example.com \ +FLAKEMETRY_TOKEN=fmk_xxx \ +FLAKEMETRY_PROJECT=acme/web \ + npx playwright test +``` + +Off CI or without a token the reporter is a no-op on delivery (fail-open). Set `FLAKEMETRY_OUTPUT_FILE=batch.json` to write the ingest batch to disk for inspection. diff --git a/packages/reporter/examples/playwright.config.ts b/packages/reporter/examples/playwright.config.ts new file mode 100644 index 0000000..a7dc139 --- /dev/null +++ b/packages/reporter/examples/playwright.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + reporter: [['@flakemetry/playwright-reporter']], +}) diff --git a/packages/reporter/package.json b/packages/reporter/package.json index c54d57d..036b406 100644 --- a/packages/reporter/package.json +++ b/packages/reporter/package.json @@ -32,11 +32,22 @@ "build": "tsup", "typecheck": "tsc -p tsconfig.json", "lint": "eslint .", - "clean": "rm -rf dist .turbo" + "clean": "rm -rf dist .turbo", + "test": "vitest run" }, "devDependencies": { "@flakemetry/eslint-config": "workspace:*", "@flakemetry/tsconfig": "workspace:*", - "@flakemetry/tsup-config": "workspace:*" + "@flakemetry/tsup-config": "workspace:*", + "@playwright/test": "^1.49.1", + "@types/node": "^22.10.5", + "vitest": "^3.2.4" + }, + "dependencies": { + "@flakemetry/contracts": "workspace:*", + "@flakemetry/sdk": "workspace:*" + }, + "peerDependencies": { + "@playwright/test": ">=1.44" } } diff --git a/packages/reporter/src/__tests__/mapping.test.ts b/packages/reporter/src/__tests__/mapping.test.ts new file mode 100644 index 0000000..9a5f7cd --- /dev/null +++ b/packages/reporter/src/__tests__/mapping.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest' + +import { buildIdempotencyKey, deriveSuite, resolveRunContext, statusFromResult } from '../mapping' + +describe('statusFromResult', () => { + it('maps a first-attempt pass to pass and a retried pass to flaky', () => { + expect(statusFromResult('passed', 0)).toBe('pass') + expect(statusFromResult('passed', 1)).toBe('flaky') + }) + + it('maps every failure kind to fail', () => { + expect(statusFromResult('failed', 0)).toBe('fail') + expect(statusFromResult('timedOut', 0)).toBe('fail') + expect(statusFromResult('interrupted', 0)).toBe('fail') + }) + + it('maps skipped to skip', () => { + expect(statusFromResult('skipped', 0)).toBe('skip') + }) +}) + +describe('deriveSuite', () => { + it('joins only describe titles, ignoring root/project/file nodes', () => { + const suite = deriveSuite([ + { type: 'root', title: '' }, + { type: 'project', title: 'chromium' }, + { type: 'file', title: 'login.spec.ts' }, + { type: 'describe', title: 'auth' }, + { type: 'describe', title: 'login' }, + ]) + expect(suite).toBe('auth > login') + }) + + it('is empty when there are no describe blocks', () => { + expect(deriveSuite([{ type: 'file', title: 'a.spec.ts' }])).toBe('') + }) +}) + +describe('resolveRunContext', () => { + it('reads github actions context including pr number from the ref', () => { + const context = resolveRunContext({ + GITHUB_ACTIONS: 'true', + GITHUB_SHA: 'abc1234', + GITHUB_REF_NAME: 'feat/login', + GITHUB_RUN_ID: '9000001', + GITHUB_EVENT_NAME: 'pull_request', + GITHUB_REF: 'refs/pull/42/merge', + FLAKEMETRY_PROJECT: 'acme/web', + }) + expect(context.ciProvider).toBe('github_actions') + expect(context.trigger).toBe('pull_request') + expect(context.commitSha).toBe('abc1234') + expect(context.prNumber).toBe(42) + expect(context.project).toBe('acme/web') + }) + + it('falls back to local defaults off CI', () => { + const context = resolveRunContext({}) + expect(context.ciProvider).toBe('local') + expect(context.trigger).toBe('manual') + expect(context.prNumber).toBeNull() + }) +}) + +describe('buildIdempotencyKey', () => { + const base = { + project: 'acme/web', + commitSha: 'abc', + branch: 'main', + ciProvider: 'github_actions' as const, + trigger: 'push' as const, + ciRunId: '9000001', + prNumber: null, + } + + it('derives a stable key from the ci run and attempt', () => { + expect(buildIdempotencyKey(base, { GITHUB_RUN_ATTEMPT: '2' })).toBe('github_actions-9000001-2') + }) + + it('honors an explicit override', () => { + expect(buildIdempotencyKey(base, { FLAKEMETRY_IDEMPOTENCY_KEY: 'custom-key-1234' })).toBe( + 'custom-key-1234', + ) + }) + + it('generates a local key when there is no ci run id', () => { + const key = buildIdempotencyKey({ ...base, ciRunId: null }, {}) + expect(key.startsWith('local-')).toBe(true) + expect(key.length).toBeGreaterThanOrEqual(8) + }) +}) diff --git a/packages/reporter/src/__tests__/reporter.test.ts b/packages/reporter/src/__tests__/reporter.test.ts new file mode 100644 index 0000000..c99e4ee --- /dev/null +++ b/packages/reporter/src/__tests__/reporter.test.ts @@ -0,0 +1,110 @@ +import { readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { ingestRunBatchSchema } from '@flakemetry/contracts' +import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '@playwright/test/reporter' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import FlakemetryReporter from '../reporter' + +const rootDir = '/repo' + +const makeSuiteChain = (describeTitle: string): Suite => { + const root = { type: 'root', title: '', parent: undefined } as unknown as Suite + const file = { type: 'file', title: 'login.spec.ts', parent: root } as unknown as Suite + return { type: 'describe', title: describeTitle, parent: file } as unknown as Suite +} + +const makeTest = (id: string, title: string, describeTitle: string): TestCase => + ({ + id, + title, + location: { file: `${rootDir}/e2e/login.spec.ts`, line: 1, column: 1 }, + parent: makeSuiteChain(describeTitle), + }) as unknown as TestCase + +const makeResult = ( + status: TestResult['status'], + retry: number, + error?: TestResult['error'], +): TestResult => + ({ + status, + retry, + duration: 1500, + startTime: new Date('2026-07-16T10:00:00Z'), + error, + }) as unknown as TestResult + +const drive = async (outputFile: string) => { + const reporter = new FlakemetryReporter({ outputFile }) + reporter.onBegin({ rootDir } as FullConfig, {} as Suite) + + const pass = makeTest('t1', 'logs in', 'auth') + reporter.onTestEnd(pass, makeResult('passed', 0)) + + const fail = makeTest('t2', 'rejects invalid', 'auth') + reporter.onTestEnd( + fail, + makeResult('failed', 0, { + message: 'expected true', + stack: 'at login:5', + } as TestResult['error']), + ) + + const flaky = makeTest('t3', 'completes payment', 'checkout') + reporter.onTestEnd( + flaky, + makeResult('failed', 0, { message: 'race', stack: 'at pay:9' } as TestResult['error']), + ) + reporter.onTestEnd(flaky, makeResult('passed', 1)) + + await reporter.onEnd({ status: 'failed' } as FullResult) +} + +describe('FlakemetryReporter lifecycle', () => { + beforeEach(() => { + vi.stubEnv('GITHUB_ACTIONS', '') + vi.stubEnv('FLAKEMETRY_PROJECT', 'acme/web') + vi.stubEnv('FLAKEMETRY_COMMIT_SHA', 'deadbeef') + vi.stubEnv('FLAKEMETRY_ENDPOINT', '') + vi.stubEnv('FLAKEMETRY_TOKEN', '') + }) + + afterEach(() => { + vi.unstubAllEnvs() + }) + + it('produces a contract-valid batch from a full run', async () => { + const outputFile = join(tmpdir(), `flakemetry-reporter-${process.hrtime.bigint()}.json`) + await drive(outputFile) + + const raw = JSON.parse(readFileSync(outputFile, 'utf8')) + const batch = ingestRunBatchSchema.parse(raw) + + expect(batch.resource.commitSha).toBe('deadbeef') + expect(batch.resource.ciProvider).toBe('local') + expect(batch.run.status).toBe('failed') + expect(batch.executions).toHaveLength(4) + }) + + it('maps statuses and links the flaky retry to its first attempt', async () => { + const outputFile = join(tmpdir(), `flakemetry-reporter-${process.hrtime.bigint()}.json`) + await drive(outputFile) + + const batch = ingestRunBatchSchema.parse(JSON.parse(readFileSync(outputFile, 'utf8'))) + const [pass, fail, flakyFirst, flakyRetry] = batch.executions + + expect(pass?.status).toBe('pass') + expect(fail?.status).toBe('fail') + expect(fail?.error?.message).toBe('expected true') + expect(flakyFirst?.status).toBe('fail') + expect(flakyFirst?.attempt).toBe(1) + expect(flakyRetry?.status).toBe('flaky') + expect(flakyRetry?.attempt).toBe(2) + expect(flakyRetry?.retryOfIndex).toBe(2) + expect(batch.executions[1]?.suite).toBe('auth') + expect(flakyRetry?.suite).toBe('checkout') + }) +}) diff --git a/packages/reporter/src/index.ts b/packages/reporter/src/index.ts index 49f298a..3fa544d 100644 --- a/packages/reporter/src/index.ts +++ b/packages/reporter/src/index.ts @@ -1,6 +1,3 @@ -export const PACKAGE = '@flakemetry/playwright-reporter' as const - -export const reporter = { - name: PACKAGE, - version: '0.0.0', -} as const +export * from './mapping' +export type { FlakemetryReporterOptions } from './reporter' +export { default } from './reporter' diff --git a/packages/reporter/src/mapping.ts b/packages/reporter/src/mapping.ts new file mode 100644 index 0000000..8325ef6 --- /dev/null +++ b/packages/reporter/src/mapping.ts @@ -0,0 +1,62 @@ +import { randomUUID } from 'node:crypto' + +import type { CiProvider, RunTrigger, TestStatus } from '@flakemetry/contracts' +import type { RunContext } from '@flakemetry/sdk' + +export type PlaywrightStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted' + +export const statusFromResult = (status: PlaywrightStatus, retry: number): TestStatus => { + if (status === 'skipped') return 'skip' + if (status === 'passed') return retry > 0 ? 'flaky' : 'pass' + return 'fail' +} + +export interface SuiteNode { + type: string + title: string +} + +export const deriveSuite = (ancestors: readonly SuiteNode[]): string => + ancestors + .filter((node) => node.type === 'describe' && node.title.length > 0) + .map((node) => node.title) + .join(' > ') + +const prNumberFromRef = (ref: string | undefined): number | null => { + if (!ref) return null + const match = /refs\/pull\/(\d+)\//.exec(ref) + return match ? Number(match[1]) : null +} + +export const resolveRunContext = (env: Record): RunContext => { + const onGithub = env.GITHUB_ACTIONS === 'true' + const ciProvider: CiProvider = onGithub ? 'github_actions' : 'local' + const trigger: RunTrigger = onGithub + ? env.GITHUB_EVENT_NAME === 'pull_request' + ? 'pull_request' + : env.GITHUB_EVENT_NAME === 'schedule' + ? 'schedule' + : 'push' + : 'manual' + + return { + project: env.FLAKEMETRY_PROJECT ?? 'local/project', + commitSha: env.GITHUB_SHA ?? env.FLAKEMETRY_COMMIT_SHA ?? '0000000', + branch: env.GITHUB_REF_NAME ?? env.FLAKEMETRY_BRANCH ?? 'local', + ciProvider, + trigger, + ciRunId: env.GITHUB_RUN_ID ?? null, + prNumber: prNumberFromRef(env.GITHUB_REF), + } +} + +export const buildIdempotencyKey = ( + context: RunContext, + env: Record, +): string => { + const explicit = env.FLAKEMETRY_IDEMPOTENCY_KEY + if (explicit) return explicit + if (context.ciRunId) + return `${context.ciProvider}-${context.ciRunId}-${env.GITHUB_RUN_ATTEMPT ?? '1'}` + return `local-${randomUUID()}` +} diff --git a/packages/reporter/src/reporter.ts b/packages/reporter/src/reporter.ts new file mode 100644 index 0000000..7ed78ce --- /dev/null +++ b/packages/reporter/src/reporter.ts @@ -0,0 +1,116 @@ +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, relative } from 'node:path' + +import type { IngestRunBatch } from '@flakemetry/contracts' +import { IngestClient, type RunContext, TestRunRecorder } from '@flakemetry/sdk' +import type { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestResult, +} from '@playwright/test/reporter' + +import { + buildIdempotencyKey, + deriveSuite, + type PlaywrightStatus, + resolveRunContext, + statusFromResult, + type SuiteNode, +} from './mapping' + +export interface FlakemetryReporterOptions { + endpoint?: string + token?: string + outputFile?: string +} + +const collectAncestors = (test: TestCase): SuiteNode[] => { + const ancestors: SuiteNode[] = [] + let current: Suite | undefined = test.parent + while (current) { + ancestors.unshift({ type: current.type, title: current.title }) + current = current.parent + } + return ancestors +} + +export default class FlakemetryReporter implements Reporter { + private readonly options: FlakemetryReporterOptions + private readonly env: Record + private recorder: TestRunRecorder | null = null + private context: RunContext | null = null + private rootDir = process.cwd() + private readonly firstAttemptIndex = new Map() + + constructor(options: FlakemetryReporterOptions = {}) { + this.options = options + this.env = process.env + } + + onBegin(config: FullConfig, _suite: Suite): void { + this.rootDir = config.rootDir + this.context = resolveRunContext(this.env) + this.recorder = new TestRunRecorder(this.context) + this.recorder.startRun(new Date()) + } + + onTestEnd(test: TestCase, result: TestResult): void { + if (!this.recorder) return + const index = this.recorder.recorded.length + const attempt = result.retry + 1 + const retryOfIndex = attempt > 1 ? (this.firstAttemptIndex.get(test.id) ?? null) : null + const error = result.error + ? { + type: result.error.value ?? undefined, + message: result.error.message ?? 'unknown error', + stack: result.error.stack ?? undefined, + } + : null + + this.recorder.record({ + filePath: relative(this.rootDir, test.location.file), + suite: deriveSuite(collectAncestors(test)), + title: test.title, + status: statusFromResult(result.status as PlaywrightStatus, result.retry), + attempt, + retryOfIndex, + startedAt: result.startTime, + durationMs: Math.round(result.duration), + error, + }) + + if (attempt === 1) this.firstAttemptIndex.set(test.id, index) + } + + async onEnd(result: FullResult): Promise { + if (!this.recorder || !this.context) return + this.recorder.finishRun(result.status === 'passed' ? 'passed' : 'failed', new Date()) + const batch = this.recorder.toIngestBatch(buildIdempotencyKey(this.context, this.env)) + + this.writeOutput(batch) + await this.deliver(batch) + } + + private writeOutput(batch: IngestRunBatch): void { + const outputFile = this.options.outputFile ?? this.env.FLAKEMETRY_OUTPUT_FILE + if (!outputFile) return + mkdirSync(dirname(outputFile), { recursive: true }) + writeFileSync(outputFile, JSON.stringify(batch, null, 2)) + } + + private async deliver(batch: IngestRunBatch): Promise { + const endpoint = this.options.endpoint ?? this.env.FLAKEMETRY_ENDPOINT + const token = this.options.token ?? this.env.FLAKEMETRY_TOKEN + if (!endpoint || !token) return + const client = new IngestClient({ endpoint, token }) + const outcome = await client.send(batch) + if (!outcome.ok) { + process.stderr.write( + `flakemetry: upload skipped (${outcome.error ?? `status ${outcome.status}`})\n`, + ) + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 33d5c98..c65cd79 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -183,6 +183,13 @@ importers: version: 8.64.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) packages/reporter: + dependencies: + '@flakemetry/contracts': + specifier: workspace:* + version: link:../contracts + '@flakemetry/sdk': + specifier: workspace:* + version: link:../sdk devDependencies: '@flakemetry/eslint-config': specifier: workspace:* @@ -193,6 +200,15 @@ importers: '@flakemetry/tsup-config': specifier: workspace:* version: link:../tsup-config + '@playwright/test': + specifier: ^1.49.1 + version: 1.61.1 + '@types/node': + specifier: ^22.10.5 + version: 22.20.1 + vitest: + specifier: ^3.2.4 + version: 3.2.7(@types/node@22.20.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0) packages/sdk: dependencies: @@ -742,6 +758,11 @@ packages: resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} engines: {node: '>=14'} + '@playwright/test@1.61.1': + resolution: {integrity: sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==} + engines: {node: '>=18'} + hasBin: true + '@prisma/client@6.19.3': resolution: {integrity: sha512-mKq3jQFhjvko5LTJFHGilsuQs+W+T3Gm451NzuTDGQxwCzwXHYnIu2zGkRoW+Exq3Rob7yp2MfzSrdIiZVhrBg==} engines: {node: '>=18.18'} @@ -1383,6 +1404,11 @@ packages: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1671,6 +1697,16 @@ packages: pkg-types@2.3.1: resolution: {integrity: sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg==} + playwright-core@1.61.1: + resolution: {integrity: sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.61.1: + resolution: {integrity: sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==} + engines: {node: '>=18'} + hasBin: true + postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -2486,6 +2522,10 @@ snapshots: '@opentelemetry/semantic-conventions@1.43.0': {} + '@playwright/test@1.61.1': + dependencies: + playwright: 1.61.1 + '@prisma/client@6.19.3(prisma@6.19.3(typescript@5.9.3))(typescript@5.9.3)': optionalDependencies: prisma: 6.19.3(typescript@5.9.3) @@ -3150,6 +3190,9 @@ snapshots: jsonfile: 4.0.0 universalify: 0.1.2 + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -3401,6 +3444,14 @@ snapshots: exsolve: 1.1.0 pathe: 2.0.3 + playwright-core@1.61.1: {} + + playwright@1.61.1: + dependencies: + playwright-core: 1.61.1 + optionalDependencies: + fsevents: 2.3.2 + postcss-load-config@6.0.1(jiti@2.7.0)(postcss@8.5.19)(tsx@4.23.1)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 From cc3b01ec1dac9664ee2ca28d21761bb60b91ae28 Mon Sep 17 00:00:00 2001 From: Andrii Kohut Date: Fri, 17 Jul 2026 12:44:39 +0200 Subject: [PATCH 2/2] fix: treat empty-string ci env vars as absent in run context The reporter lifecycle test leaked GITHUB_SHA from the CI environment because resolveRunContext used nullish coalescing, which does not fall through on empty strings. Normalize empty env vars to undefined so set-but-empty CI variables fall back correctly, and make the lifecycle test hermetic by stubbing all GitHub env vars. --- packages/reporter/src/__tests__/mapping.test.ts | 10 ++++++++++ .../reporter/src/__tests__/reporter.test.ts | 17 ++++++++++++++--- packages/reporter/src/mapping.ts | 13 ++++++++----- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/packages/reporter/src/__tests__/mapping.test.ts b/packages/reporter/src/__tests__/mapping.test.ts index 9a5f7cd..4b11ec4 100644 --- a/packages/reporter/src/__tests__/mapping.test.ts +++ b/packages/reporter/src/__tests__/mapping.test.ts @@ -60,6 +60,16 @@ describe('resolveRunContext', () => { expect(context.trigger).toBe('manual') expect(context.prNumber).toBeNull() }) + + it('treats empty-string env vars as absent', () => { + const context = resolveRunContext({ + GITHUB_SHA: '', + GITHUB_REF_NAME: '', + FLAKEMETRY_COMMIT_SHA: 'deadbeef', + }) + expect(context.commitSha).toBe('deadbeef') + expect(context.branch).toBe('local') + }) }) describe('buildIdempotencyKey', () => { diff --git a/packages/reporter/src/__tests__/reporter.test.ts b/packages/reporter/src/__tests__/reporter.test.ts index c99e4ee..3184d76 100644 --- a/packages/reporter/src/__tests__/reporter.test.ts +++ b/packages/reporter/src/__tests__/reporter.test.ts @@ -65,11 +65,22 @@ const drive = async (outputFile: string) => { describe('FlakemetryReporter lifecycle', () => { beforeEach(() => { - vi.stubEnv('GITHUB_ACTIONS', '') + for (const key of [ + 'GITHUB_ACTIONS', + 'GITHUB_SHA', + 'GITHUB_REF', + 'GITHUB_REF_NAME', + 'GITHUB_RUN_ID', + 'GITHUB_RUN_ATTEMPT', + 'GITHUB_EVENT_NAME', + 'FLAKEMETRY_ENDPOINT', + 'FLAKEMETRY_TOKEN', + 'FLAKEMETRY_IDEMPOTENCY_KEY', + ]) { + vi.stubEnv(key, '') + } vi.stubEnv('FLAKEMETRY_PROJECT', 'acme/web') vi.stubEnv('FLAKEMETRY_COMMIT_SHA', 'deadbeef') - vi.stubEnv('FLAKEMETRY_ENDPOINT', '') - vi.stubEnv('FLAKEMETRY_TOKEN', '') }) afterEach(() => { diff --git a/packages/reporter/src/mapping.ts b/packages/reporter/src/mapping.ts index 8325ef6..dfa5a2c 100644 --- a/packages/reporter/src/mapping.ts +++ b/packages/reporter/src/mapping.ts @@ -28,6 +28,9 @@ const prNumberFromRef = (ref: string | undefined): number | null => { return match ? Number(match[1]) : null } +const pick = (value: string | undefined): string | undefined => + value && value.length > 0 ? value : undefined + export const resolveRunContext = (env: Record): RunContext => { const onGithub = env.GITHUB_ACTIONS === 'true' const ciProvider: CiProvider = onGithub ? 'github_actions' : 'local' @@ -40,13 +43,13 @@ export const resolveRunContext = (env: Record): RunC : 'manual' return { - project: env.FLAKEMETRY_PROJECT ?? 'local/project', - commitSha: env.GITHUB_SHA ?? env.FLAKEMETRY_COMMIT_SHA ?? '0000000', - branch: env.GITHUB_REF_NAME ?? env.FLAKEMETRY_BRANCH ?? 'local', + project: pick(env.FLAKEMETRY_PROJECT) ?? 'local/project', + commitSha: pick(env.GITHUB_SHA) ?? pick(env.FLAKEMETRY_COMMIT_SHA) ?? '0000000', + branch: pick(env.GITHUB_REF_NAME) ?? pick(env.FLAKEMETRY_BRANCH) ?? 'local', ciProvider, trigger, - ciRunId: env.GITHUB_RUN_ID ?? null, - prNumber: prNumberFromRef(env.GITHUB_REF), + ciRunId: pick(env.GITHUB_RUN_ID) ?? null, + prNumber: prNumberFromRef(pick(env.GITHUB_REF)), } }