fix(website-lambda): gate .plain.html and .md variants of private pages

workers/website-lambda/README.md

@@ -70,9 +70,32 @@ dynamically — the `nodejs22.x` runtime provides AWS SDK v3, so it is not bundl
 ever lacks it, the dynamic import degrades to fail-closed instead of crashing init; the fallback is
 to `npm i` it and include `node_modules` in the deploy zip.
 
+## Page variants (`.plain.html` and `.md`)
+
+AEM serves every page in two head-less variants as well: `<page>.plain.html`
+(body markup only) and `<page>.md` (Markdown). Neither includes the page's
+`<head>`, so the `<meta name="audience" content="private">` check can't run on
+the variant itself. For anonymous visitors the Lambda handles them as follows:
+
+1. [`lib/gate.js`](./lib/gate.js) `getCanonicalPagePath` maps the variant to its
+   page (`/a/b.plain.html` → `/a/b`, `/index.md` → `/`, `/a/index.md` → `/a/`).
+2. Before proxying the variant, [`index.js`](./index.js) `isPublicCanonicalPage`
+   fetches that page from AEM and runs `isPrivateHtml` on it. The lookup fails
+   closed: a private page, a non-`200` or redirect, a non-HTML response, or a
+   fetch error returns `404`, and the variant is never fetched.
+3. A public variant is then served with its audience blocks stripped:
+   `filterAudienceBlocks` handles `.plain.html` markup (no `<main>` wrapper), and
+   `filterAudienceMarkdown` removes `Name (audience private)` block tables from
+   `.md`.
+
+Authenticated requests skip the canonical lookup. For anonymous requests it
+adds one origin fetch per variant request that reaches the Lambda; the result is
+cached like any other anonymous page (see "Content caching"). Both variants fall
+under the default CloudFront behavior, so they always reach the Lambda.
+
 ## CloudFront routing
 
-Only HTML and JSON vary by viewer, so only they go through the Lambda. Everything
+Only HTML, Markdown (`.md`), and JSON vary by viewer, so only they go through the Lambda. Everything
 else goes straight from CloudFront to AEM.
 
 | Behavior (in order) | Origin | Cache policy (prod) | Cache policy (stage) |

workers/website-lambda/index.js

@@ -35,8 +35,10 @@
 import { fetchFromAem } from './handlers/aem.js';
 import { createSession, deleteSession } from './handlers/auth.js';
 import { readSession, DEFAULT_SESSION_COOKIE_NAME } from './lib/session.js';
-import { classifyPublicPath, isPageLike, isPrivateHtml, PUBLIC_FILTER_PATHS } from './lib/gate.js';
-import { filterAudienceBlocks } from './lib/audience.js';
+import {
+  classifyPublicPath, getCanonicalPagePath, isPageLike, isPrivateHtml, PUBLIC_FILTER_PATHS,
+} from './lib/gate.js';
+import { filterAudienceBlocks, filterAudienceMarkdown } from './lib/audience.js';
 import { filterPrivateEntries, compactEntries, collectPrivatePaths } from './lib/query-index.js';
 import { filterSitemap, rewriteSitemapHosts } from './lib/sitemap.js';
 import { toGatedEtag, toUpstreamIfNoneMatch } from './lib/etag.js';
@@ -213,21 +215,23 @@ const isAuthenticated = async (request) => {
   return (await readSession(cookie, env.SESSION_SECRET, Date.now())) !== null;
 };
 
-// Post-fetch processing of a proxied HTML page, in two passes that both need
-// the body (which lives in the HTML and so can only be inspected after
-// proxying):
+// Post-fetch processing of a proxied page, in two passes that both need the
+// body (which lives in the HTML and so can only be inspected after proxying):
 //   1. Meta gate (anonymous only): a page that opts into privacy with
 //      <meta name="audience" content="private"> becomes a 404, indistinguishable
-//      from a path that does not exist. Authenticated viewers see it.
+//      from a path that does not exist. Authenticated viewers see it. Skipped
+//      for head-less variants (`variant`): a .plain.html body has no <head> to
+//      scan, so route() has already gated it on the canonical page instead.
 //   2. Audience blocks: content blocks the viewer must not see are stripped
 //      (audience-private for anonymous, audience-public for authenticated) so
-//      private markup never leaves the edge.
-// Non-HTML/non-200 responses (assets, redirects, the AEM 404 for a missing
-// page) pass through untouched without reading the body. content-length and
-// content-encoding are stripped in toLambdaResponse, so re-wrapping the
-// already-read body here stays consistent. `anonPage` marks an anonymous
-// page-like request, whose filtered response gets a gated ETag.
-const processHtmlResponse = async (resp, authed, anonPage) => {
+//      private markup never leaves the edge - from HTML (full page or
+//      .plain.html) and from the .md variant's Markdown alike.
+// Non-HTML/Markdown or non-200 responses (assets, redirects, the AEM 404 for a
+// missing page) pass through untouched without reading the body.
+// content-length and content-encoding are stripped in toLambdaResponse, so
+// re-wrapping the already-read body here stays consistent. `anonPage` marks an
+// anonymous page-like request, whose filtered response gets a gated ETag.
+const processHtmlResponse = async (resp, authed, anonPage, variant = false) => {
   const contentType = resp.headers.get('content-type') || '';
   // Revalidation of a cached anonymous page. route() only forwards a gated
   // If-None-Match (a tag this gate version issued), so this 304 confirms a copy
@@ -241,10 +245,14 @@ const processHtmlResponse = async (resp, authed, anonPage) => {
   // Only a full 200 body can be gated. Any other success (e.g. a 206 slice)
   // would skip the private-page check, so fail closed for anonymous pages.
   if (anonPage && resp.status > 200 && resp.status < 300) { return notFound(); }
-  if (resp.status !== 200 || !contentType.includes('text/html')) { return resp; }
+  const isHtml = contentType.includes('text/html');
+  const isMarkdown = contentType.includes('text/markdown');
+  if (resp.status !== 200 || !(isHtml || isMarkdown)) { return resp; }
   const body = await resp.text();
-  if (!authed && isPrivateHtml(body)) { return notFound(); }
-  const filtered = filterAudienceBlocks(body, authed);
+  if (!authed && !variant && isPrivateHtml(body)) { return notFound(); }
+  const filtered = isMarkdown
+    ? filterAudienceMarkdown(body, authed)
+    : filterAudienceBlocks(body, authed);
   const out = new Response(filtered, resp);
   // Filtered per viewer: anonymous gets the public view (short shared TTL),
   // authenticated stays no-store. The cookie-keyed cache keeps them separate.
@@ -253,6 +261,35 @@ const processHtmlResponse = async (resp, authed, anonPage) => {
   return out;
 };
 
+// Privacy check for a head-less page variant (.plain.html / .md): neither body
+// carries the page's <head>, so the audience meta is read from the canonical
+// page instead. GETs the canonical path from AEM through the same upstream
+// setup as the variant (origin, credential, no cookies), dropping conditional
+// headers so a revalidation can't turn it into a bodiless 304, and not
+// following redirects. Fails closed: only a 200 HTML canonical page without
+// the private meta clears the variant; private, missing, redirected, non-HTML,
+// or a fetch error all answer false, and the caller 404s.
+const isPublicCanonicalPage = async (req, url, canonicalPath) => {
+  try {
+    const canonicalUrl = new URL(url.href);
+    canonicalUrl.pathname = canonicalPath;
+    canonicalUrl.search = '';
+    const headers = new Headers(req.headers);
+    for (const name of ['if-none-match', 'if-modified-since', 'if-match', 'if-unmodified-since', 'if-range', 'range']) {
+      headers.delete(name);
+    }
+    const base = new Request(canonicalUrl, { method: 'GET', headers });
+    const upstream = await formatRequest(base, canonicalUrl);
+    const resp = await fetch(upstream, { cache: 'no-store', redirect: 'manual' });
+    if (resp.status !== 200) { return false; }
+    if (!(resp.headers.get('content-type') || '').includes('text/html')) { return false; }
+    return !isPrivateHtml(await resp.text());
+  } catch (err) {
+    console.error('website-lambda: canonical privacy check failed; denying variant:', err);
+    return false;
+  }
+};
+
 // Post-fetch transform for the query-index JSON: for anonymous visitors strip
 // the private rows and the whole `audience` column (so private paths/titles/
 // excerpts never reach an anonymous client, and it can't even tell which rows
@@ -432,6 +469,18 @@ const route = async (req) => {
   const verdict = authed ? 'allow' : classifyPublicPath(url.pathname);
   if (verdict === 'deny') { return notFound(); }
 
+  // Head-less page variants (.plain.html / .md) carry no <head>, so the meta
+  // gate can't run on their body. For an anonymous visitor, settle privacy on
+  // the canonical page *before* proxying the variant; a private (or
+  // unverifiable) page is a 404 and the variant is never fetched. Runs for
+  // every method, so a HEAD can't probe a private page either.
+  const canonicalPath = getCanonicalPagePath(url.pathname);
+  const variant = canonicalPath !== null;
+  if (!authed && verdict === 'gate' && variant
+    && !(await isPublicCanonicalPage(req, url, canonicalPath))) {
+    return notFound();
+  }
+
   // Read the compact opt-in before formatSearchParams strips it (it keeps only
   // limit/offset/sheet for JSON) - the query-index transform below reads it.
   const compact = url.searchParams.get('compact') === 'true';
@@ -525,10 +574,10 @@ const route = async (req) => {
 
   // Every other proxied HTML page is processed - not just 'gate' pages - so
   // allow-listed content (e.g. the homepage, which carries audience blocks) is
-  // filtered too. processHtmlResponse no-ops on non-HTML responses. `anonPage`
-  // tells it a bodiless 304 is a gated page revalidation that keeps the short
-  // anon TTL and gated tag, vs an asset 304 (leave AEM's headers).
-  return processHtmlResponse(resp, authed, anonPage);
+  // filtered too. processHtmlResponse no-ops on anything but HTML/Markdown.
+  // `anonPage` tells it a bodiless 304 is a gated page revalidation that keeps
+  // the short anon TTL and gated tag, vs an asset 304 (leave AEM's headers).
+  return processHtmlResponse(resp, authed, anonPage, variant);
 };
 
 /*

workers/website-lambda/index.variants.test.js

@@ -0,0 +1,217 @@
+import {
+  describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach,
+} from 'vitest';
+import { signToken } from './lib/session.js';
+import { GATE_ETAG_SUFFIX } from './lib/etag.js';
+
+// Routing tests for the head-less page variant gate (.plain.html / .md). The
+// global fetch is stubbed with a fake AEM origin keyed by path, so both the
+// proxied variant and the canonical-page privacy lookup stay local - no
+// network, no real session tokens.
+
+const TEST_ENV = {
+  AEM_ORG: 'adobe',
+  AEM_SITE: 'spectrum-hub',
+  SESSION_SECRET: 'test-secret',
+};
+let savedEnv;
+let handler;
+
+beforeAll(async () => {
+  savedEnv = Object.fromEntries(Object.keys(TEST_ENV).map((key) => [key, process.env[key]]));
+  Object.assign(process.env, TEST_ENV);
+  ({ handler } = await import('./index.js'));
+});
+
+afterAll(() => {
+  for (const [key, value] of Object.entries(savedEnv)) {
+    if (value === undefined) { delete process.env[key]; } else { process.env[key] = value; }
+  }
+});
+
+const html = (head, main) => `<!doctype html><html><head>${head}</head><body><main>${main}</main></body></html>`;
+const PRIVATE_META = '<meta name="audience" content="private">';
+const PRIVATE_BLOCK = '<div class="banner audience-private">private banner</div>';
+const PUBLIC_BLOCK = '<div class="banner audience-public">public banner</div>';
+
+const mdTable = (header, body) => {
+  const width = Math.max(header.length, body.length) + 2;
+  const border = `+${'-'.repeat(width)}+`;
+  const row = (text) => `| ${text.padEnd(width - 2)} |`;
+  return [border, row(header), border, row(body), border].join('\n');
+};
+
+const HTML_TYPE = { 'content-type': 'text/html; charset=utf-8' };
+const MD_TYPE = { 'content-type': 'text/markdown; charset=utf-8' };
+
+// The fake AEM origin: path -> () => Response.
+const PAGES = {
+  '/private': () => new Response(html(PRIVATE_META, '<div><p>secret</p></div>'), { headers: HTML_TYPE }),
+  '/private.plain.html': () => new Response('<div><p>secret</p></div>', { headers: HTML_TYPE }),
+  '/private.md': () => new Response(`# Secret\n\n${mdTable('Metadata', 'audience | private')}\n`, { headers: MD_TYPE }),
+  '/': () => new Response(html('', `<div>${PUBLIC_BLOCK}${PRIVATE_BLOCK}</div>`), { headers: HTML_TYPE }),
+  '/index.plain.html': () => new Response(`<div>${PUBLIC_BLOCK}${PRIVATE_BLOCK}</div>`, { headers: HTML_TYPE }),
+  '/index.md': () => new Response(
+    `${mdTable('Banner (audience public)', 'public banner')}\n\n${mdTable('Banner (audience private)', 'private banner')}\n\n# Home\n`,
+    { headers: MD_TYPE },
+  ),
+  '/public': () => new Response(html('', '<div><p>hello</p></div>'), { headers: HTML_TYPE }),
+  '/public.plain.html': () => new Response('<div><p>hello</p></div>', { headers: HTML_TYPE }),
+  '/moved': () => new Response(null, { status: 301, headers: { location: '/elsewhere' } }),
+  '/moved.plain.html': () => new Response('<div><p>moved</p></div>', { headers: HTML_TYPE }),
+  '/broken': () => new Response('upstream error', { status: 500 }),
+  '/broken.plain.html': () => new Response('<div><p>broken</p></div>', { headers: HTML_TYPE }),
+  '/orphan.plain.html': () => new Response('<div><p>orphan</p></div>', { headers: HTML_TYPE }),
+};
+
+let fetchMock;
+
+const upstreamPaths = () => fetchMock.mock.calls.map(([req]) => new URL(req.url).pathname);
+
+const sessionCookie = async () => {
+  const claims = JSON.stringify({
+    email: 'user@example.com',
+    created_at: String(Date.now()),
+    expires_in: '86400000',
+  });
+  return `spectrum_session=${await signToken(claims, TEST_ENV.SESSION_SECRET)}`;
+};
+
+const invoke = async (path, { method = 'GET', cookies, headers = {} } = {}) => {
+  const resp = await handler({
+    rawPath: path,
+    rawQueryString: '',
+    headers: { host: 'example.com', ...headers },
+    cookies,
+    requestContext: { http: { method } },
+  });
+  const body = resp.isBase64Encoded ? Buffer.from(resp.body, 'base64').toString('utf8') : resp.body;
+  return { ...resp, text: body };
+};
+
+beforeEach(() => {
+  fetchMock = vi.fn(async (req) => {
+    const page = PAGES[new URL(req.url).pathname];
+    return page ? page() : new Response('Not found', { status: 404, headers: HTML_TYPE });
+  });
+  vi.stubGlobal('fetch', fetchMock);
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe('private page variants (anonymous)', () => {
+  it('still 404s the extensionless private page', async () => {
+    const resp = await invoke('/private');
+    expect(resp.statusCode).toBe(404);
+  });
+
+  it('404s the .plain.html of a private page without fetching the variant', async () => {
+    const resp = await invoke('/private.plain.html');
+    expect(resp.statusCode).toBe(404);
+    expect(resp.text).not.toContain('secret');
+    expect(resp.headers['cache-control']).toBe('no-store');
+    expect(upstreamPaths()).toEqual(['/private']);
+  });
+
+  it('404s the .md of a private page', async () => {
+    const resp = await invoke('/private.md');
+    expect(resp.statusCode).toBe(404);
+    expect(resp.text).not.toContain('Secret');
+    expect(upstreamPaths()).toEqual(['/private']);
+  });
+
+  it('404s a HEAD for a private variant too', async () => {
+    const resp = await invoke('/private.plain.html', { method: 'HEAD' });
+    expect(resp.statusCode).toBe(404);
+  });
+
+  it('checks the canonical page with a plain GET, no conditional headers or cookies', async () => {
+    await invoke('/private.plain.html', {
+      method: 'HEAD',
+      headers: { 'if-none-match': '"abc"', 'if-modified-since': 'Wed, 01 Jan 2025 00:00:00 GMT' },
+      cookies: ['other=1'],
+    });
+    const [[canonicalReq, init]] = fetchMock.mock.calls;
+    expect(new URL(canonicalReq.url).pathname).toBe('/private');
+    expect(canonicalReq.method).toBe('GET');
+    expect(canonicalReq.headers.get('if-none-match')).toBeNull();
+    expect(canonicalReq.headers.get('if-modified-since')).toBeNull();
+    expect(canonicalReq.headers.get('cookie')).toBeNull();
+    expect(new URL(canonicalReq.url).hostname).toBe('main--spectrum-hub--adobe.aem.live');
+    expect(init.redirect).toBe('manual');
+  });
+
+  it('re-checks the canonical page even when revalidating a gated copy', async () => {
+    const resp = await invoke('/private.plain.html', {
+      headers: { 'if-none-match': `W/"abc${GATE_ETAG_SUFFIX}"` },
+    });
+    expect(resp.statusCode).toBe(404);
+    expect(upstreamPaths()).toEqual(['/private']);
+  });
+
+  it.each([
+    ['the canonical page is missing', '/orphan.plain.html'],
+    ['the canonical page redirects', '/moved.plain.html'],
+    ['the canonical lookup errors upstream', '/broken.plain.html'],
+  ])('fails closed when %s', async (_label, path) => {
+    const resp = await invoke(path);
+    expect(resp.statusCode).toBe(404);
+    expect(upstreamPaths()).toHaveLength(1);
+  });
+
+  it('fails closed when the canonical fetch throws', async () => {
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+    fetchMock.mockImplementationOnce(async () => { throw new Error('network down'); });
+    const resp = await invoke('/public.plain.html');
+    expect(resp.statusCode).toBe(404);
+    errorSpy.mockRestore();
+  });
+});
+
+describe('public page variants (anonymous)', () => {
+  it('serves the .plain.html of a public page after checking the canonical page', async () => {
+    const resp = await invoke('/public.plain.html');
+    expect(resp.statusCode).toBe(200);
+    expect(resp.text).toContain('hello');
+    expect(upstreamPaths()).toEqual(['/public', '/public.plain.html']);
+  });
+
+  it('maps /index.plain.html to / and strips audience-private blocks', async () => {
+    const resp = await invoke('/index.plain.html');
+    expect(resp.statusCode).toBe(200);
+    expect(upstreamPaths()[0]).toBe('/');
+    expect(resp.text).toContain('public banner');
+    expect(resp.text).not.toContain('private banner');
+  });
+
+  it('maps /index.md to / and strips audience-private block tables', async () => {
+    const resp = await invoke('/index.md');
+    expect(resp.statusCode).toBe(200);
+    expect(upstreamPaths()[0]).toBe('/');
+    expect(resp.text).toContain('public banner');
+    expect(resp.text).not.toContain('private banner');
+    expect(resp.text).toContain('# Home');
+  });
+});
+
+describe('page variants (authenticated)', () => {
+  it('serves a private variant without a canonical lookup', async () => {
+    const resp = await invoke('/private.plain.html', { cookies: [await sessionCookie()] });
+    expect(resp.statusCode).toBe(200);
+    expect(resp.text).toContain('secret');
+    expect(resp.headers['cache-control']).toBe('private, no-store');
+    expect(upstreamPaths()).toEqual(['/private.plain.html']);
+  });
+
+  it('strips audience-public blocks from .plain.html and .md', async () => {
+    const cookies = [await sessionCookie()];
+    const plain = await invoke('/index.plain.html', { cookies });
+    expect(plain.text).toContain('private banner');
+    expect(plain.text).not.toContain('public banner');
+    const md = await invoke('/index.md', { cookies });
+    expect(md.text).toContain('private banner');
+    expect(md.text).not.toContain('public banner');
+  });
+});