ci.yml

---
name: CI

on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:

permissions:
  contents: read
  # ghcr.io reads only, so the validate job's image pulls are authenticated
  # rather than anonymous (#602). Nothing here writes a package.
  packages: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# There is no `env: STACK:` here any more. It pinned every step below to
# stacks/observability, which is why stacks/lab landed as a stack CI had never
# seen (#263, #264). Each step loops over ./scripts/stacks.sh instead — the one
# place that defines what a stack is, and the thing that fails when a directory
# under stacks/ has no compose.yaml.
#
# Image versions are NOT duplicated here either. They are resolved from
# compose.yaml at run time by scripts/image-for.sh, because Dependabot only
# updates compose.yaml — hardcoded copies went stale silently and CI ended up
# validating v3.1.0 configs against a stack running v3.13.2. gitleaks was the
# last exception to that and is now a profile-gated service in compose.yaml
# like every other image (#65).

jobs:
  # ---------------------------------------------------------------------------
  lint:
    name: Lint
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # The linter list lives in scripts/lint.sh, which `make lint` and
      # scripts/validate.sh also run — so a linter cannot exist in one place and
      # not the others, and --strict cannot be on here and off there. This job
      # held its own copy until #68: yamllint ran --strict here and bare
      # locally, so a warning passed `make validate` and failed the pull request
      # that followed.
      #
      # The two steps that were Actions are now pinned images from compose.yaml.
      # editorconfig-checker is why that mattered most: the action downloaded
      # the binary, added it to PATH, and never invoked it. A green step that
      # checked nothing, and nothing local to notice.
      #
      # --require-all turns "tool unreachable" from a counted skip into a
      # failure. A local run may honestly skip; CI may not, because a linter
      # skipped here is one nobody will ever run.
      #
      # lint.sh asserts that this job contains this step and a checkout and
      # nothing else. Adding a linter here rather than there fails, by design.
      - name: Lint
        run: ./scripts/lint.sh --require-all

  # ---------------------------------------------------------------------------
  validate:
    name: Validate configs
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # ghcr.io rate-limits by the identity asking, and an anonymous pull from
      # a runner is identified by an egress IP shared with everything else on
      # it. The probe step below pulls every pinned image in every stack on
      # every pull request — six minutes of registry traffic — and twice that
      # volume tripped the limit on diffs that were Markdown only (#602). The
      # retry in check_compose_health.py fired all three times and the fourth
      # attempt still failed, so more backoff is not the lever: fewer anonymous
      # requests is.
      #
      # GITHUB_TOKEN is scoped to this run and to `packages: read` above, which
      # is all a pull needs. A plain docker login rather than an action, so
      # this adds no dependency to pin.
      #
      # It does not fail the job, by design. A login that cannot happen — a
      # fork's restricted token, a registry outage — leaves the pulls anonymous,
      # which is exactly where they were before this step existed, and every
      # check below still fails loudly on an image it cannot fetch. Hard-failing
      # here would add a second way for CI to go red for a reason no diff can
      # cause, which is the thing #602 is about.
      #
      # Docker Hub is deliberately left alone: separate quota, separate
      # decision, and nothing has failed there.
      - name: Authenticate to ghcr.io so the pulls below are not anonymous
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GHCR_USER: ${{ github.actor }}
        run: |
          if printf '%s' "$GHCR_TOKEN" \
            | docker login ghcr.io -u "$GHCR_USER" --password-stdin; then
            echo "ghcr.io: authenticated as $GHCR_USER"
          else
            echo "::warning::ghcr.io login failed — the pulls below are anonymous"
            echo "::warning::and exposed to the shared-IP rate limit of #602"
          fi

      # age, for the volume round trip (#854). Not a pass-through shim, the way
      # the other backup suites stand one in: what those two suites prove is
      # that a set opens with either recipient's key and not a third's, and a
      # shim opens everything.
      - name: Install age
        run: sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends age

      # Every embedded fixture suite in the repository — the collectors that
      # parse output no host here can be asked to produce, the CA-key proof that
      # needs a mounted medium, the offsite copy that needs the second
      # recipient's. They are the only test those scripts have, and until now
      # they ran in scripts/validate.sh and in no job here, so they guarded a
      # `make validate` a contributor may not run and guarded a pull request not
      # at all. That is the #68 asymmetry again (#614), and the script asserts
      # this step exists, so deleting it fails rather than narrowing coverage.
      #
      # The suites are discovered, not listed: collect-gateway-state.sh had a
      # documented --self-test that nothing had ever called, which is what a
      # hand-kept list gets you. Discovery also means a suite added later is
      # covered here without anyone remembering to add it.
      #
      # First among the real work, because none of it needs a secret and all of
      # it finishes in seconds. Two suites do need the daemon and one image
      # pull — backup-volumes.sh and restore-volumes.sh round-trip a throwaway
      # volume through real age (#854) — and the runner has docker already;
      # age is installed in the step above. A broken fixture
      # is reported before the six minutes of registry traffic the probe step
      # spends below, which is the ordering #602 asked for when
      # check_compose_health.py's own suite had a dedicated step here. That
      # suite is discovered with the rest now, so the step is gone and this one
      # covers it.
      #
      # --require-all because a skip in CI is a check nobody runs. It is what
      # makes backup-offsite.sh's two conditional fixtures — the same-filesystem
      # and tmpfs refusals, the second of which IS the #596 regression — fail
      # here rather than print SKIP and still exit 0.
      - name: Run every embedded fixture suite
        run: ./scripts/self-tests.sh --require-all

      # The ${VAR:?} guards in compose.yaml exist so a missing secret fails
      # loudly at deploy time. CI supplies throwaway values so validation never
      # needs a decryption key. The list of variables lives in the script, which
      # scripts/validate.sh also calls — inlining it here is what let CI and the
      # local run drift apart. The .env written here stays gitignored.
      # Per stack, because the guard list is derived from each stack's own
      # compose.yaml — seeding one from another's guards proves nothing about
      # the file being validated.
      - name: Seed a validation-only .env
        run: |
          for stack in $(./scripts/stacks.sh); do
            ./scripts/seed-validation-env.sh "stacks/$stack/.env" "$stack"
          done

      # Single source of truth: whatever compose.yaml pins is what gets tested.
      - name: Resolve pinned images from compose.yaml
        run: |
          {
            echo "PROM_IMAGE=$(./scripts/image-for.sh prometheus)"
            echo "AM_IMAGE=$(./scripts/image-for.sh alertmanager)"
            echo "ALLOY_IMAGE=$(./scripts/image-for.sh alloy)"
          } >> "$GITHUB_ENV"
          ./scripts/image-for.sh prometheus
          ./scripts/image-for.sh alertmanager
          ./scripts/image-for.sh alloy

      - name: docker compose config
        run: |
          for stack in $(./scripts/stacks.sh); do
            echo "== $stack"
            docker compose -f "stacks/$stack/compose.yaml" config -q
          done

      # Guard against the duplication coming back. Any image: pin outside
      # compose.yaml is drift waiting to happen, since Dependabot cannot see it.
      #
      # This matched only prom/ and grafana/ until #65, which is how
      # zricethezav/gitleaks sat pinned in this file's own env: block. Any
      # repo/name:version fails now. The illustrative version in the header of
      # scripts/pin-digests.sh is a comment and is filtered out below.
      # The three pattern checks that used to live here — version pins outside
      # compose.yaml, floating :latest, and every image digest-pinned — are now
      # in scripts/check_image_pins.py, which runs in both places (#175). They
      # were unrunnable locally, and they belong beside the parser whose
      # docstring explains why a grep cannot see a pin that is simply absent.
      # The three checks above are all pattern matches, and #65 walked past all
      # three: `make backup` ran a bare `alpine` — no `:latest` literal, no
      # prom/ or grafana/ prefix, and not an `image:` line in compose.yaml. A
      # grep cannot match a pin that is simply absent. This parses every docker
      # run/pull/create in the Makefile, the scripts, this workflow and the
      # runbooks, and requires each one's image to resolve through
      # scripts/image-for.sh — so an image that is not in compose.yaml cannot be
      # run at all, whatever it is called. That is also what makes the digest
      # check above complete: it reads compose.yaml alone, and compose.yaml is
      # now the only place an image can come from.
      - name: Verify every docker image comes from compose.yaml
        run: python3 scripts/check_image_pins.py

      # Every stack that runs a Prometheus. A stack with rules and no unit
      # tests fails, which is stricter than the single-stack version was: rules
      # that cannot fire pass `check rules` (#63), so the moment to notice a
      # stack has none is when its rules land.
      - name: promtool check config, check rules, test rules
        run: |
          fail=0
          for stack in $(./scripts/stacks.sh); do
            sd="stacks/$stack"
            [ -f "$sd/prometheus/prometheus.yaml" ] || { echo "== $stack: no prometheus.yaml"; continue; }
            echo "== $stack"
            docker run --rm --entrypoint promtool \
              -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \
              check config "$sd/prometheus/prometheus.yaml" || fail=1

            rules=$(find "$sd/prometheus/rules" -name '*.rules.yaml' 2>/dev/null | sort)
            tests=$(find "$sd/prometheus/tests" -name '*.test.yaml' 2>/dev/null | sort)
            if [ -z "$rules" ]; then
              echo "   no alert rules"
              continue
            fi
            # shellcheck disable=SC2086
            docker run --rm --entrypoint promtool \
              -v "$PWD:/repo" -w /repo "$PROM_IMAGE" check rules $rules || fail=1
            if [ -z "$tests" ]; then
              echo "::error::$stack has alert rules and no promtool tests —"
              echo "::error::a rule that cannot fire still passes check rules (#63)"
              fail=1
              continue
            fi
            # shellcheck disable=SC2086
            docker run --rm --entrypoint promtool \
              -v "$PWD:/repo" -w /repo "$PROM_IMAGE" test rules $tests || fail=1
          done
          # The per-stack guard above refuses a stack with no tests at all; this
          # refuses a single untested rule inside a tested stack (#843).
          python3 scripts/check_rule_tests.py || fail=1
          exit $fail

      # No secret needed: the receiver URL comes from url_file, which
      # Alertmanager reads at notify time rather than at config load time.
      # Only stacks that run one. stacks/lab has no Alertmanager by decision
      # (ADR-0020), and the ROUTES table below describes the estate's tree
      # specifically.
      - name: amtool check-config
        run: |
          fail=0
          for f in $(./scripts/stacks.sh --paths | sed 's|$|/alertmanager/alertmanager.yaml|'); do
            [ -f "$f" ] || continue
            echo "== $f"
            docker run --rm --entrypoint amtool \
              -v "$PWD:/repo" -w /repo "$AM_IMAGE" check-config "$f" || fail=1
          done
          exit $fail

      # check-config proves the tree parses and that every route names a
      # receiver that exists. It does not say WHICH receiver an alert reaches,
      # and the tree is first-match-wins with no `continue` — so demoting a
      # category route below the bare `severity` routes stops it matching while
      # check-config still reports SUCCESS. Verified: that mutation passes
      # check-config and fails the assertions below.
      #
      # Same eight assertions as scripts/validate.sh. Both places, one commit:
      # this pair has drifted before (#68).
      - name: amtool config routes test
        run: |
          fail=0
          # The estate's tree. Asserted against the one stack that has an
          # Alertmanager rather than against every stack, because these eight
          # rows are that tree's routing and not a property all stacks share.
          cfg=stacks/observability/alertmanager/alertmanager.yaml
          while read -r expected labels; do
            [ -n "$expected" ] || continue
            # shellcheck disable=SC2086
            docker run --rm --entrypoint amtool \
              -v "$PWD:/repo" -w /repo "$AM_IMAGE" \
              config routes test \
              --config.file="$cfg" \
              --verify.receivers="$expected" $labels \
              || { echo "::error::expected $expected for $labels"; fail=1; }
          done <<'ROUTES'
          heartbeat,default alertname=Watchdog severity=none category=monitoring
          urgent    severity=critical category=power
          security  severity=critical category=security
          security  severity=warning category=security
          urgent    severity=critical category=availability
          default   severity=warning category=capacity
          default   severity=warning category=hardware
          null      severity=info category=correctness
          ROUTES
          exit $fail

      # `fmt --test` exits non-zero if the file is not canonically formatted, and
      # fails outright on a syntax error. It does not validate that components
      # are configured correctly — Alloy has no `validate` subcommand, so that is
      # only caught at load time on the host.
      - name: alloy fmt --test
        # Every file in the directory: the agent loads the directory, and the
        # deploy script ships a subset of it, so each file must stand alone.
        # Every *.alloy under stacks/, found rather than assumed to live under
        # one of them. stacks/lab has no alloy/ directory — it mounts the
        # estate's two files rather than copying them (ADR-0007) — so a `find`
        # here covers both today and any stack that grows its own tomorrow.
        run: |
          found=$(find stacks -path '*/alloy/*.alloy' | sort)
          [ -n "$found" ] || { echo "::error::no *.alloy under stacks/"; exit 1; }
          for f in $found; do
            docker run --rm --entrypoint alloy \
              -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \
              fmt --test "$f"
          done

      # `docker compose config` happily accepts a service_healthy dependency on
      # a service that can never report healthy — the stack then hangs at deploy
      # time with no error at all. Loki's distroless image has no wget to probe
      # with, which is exactly how that happened here.
      #
      # --probe is what makes that verified rather than assumed: it execs each
      # healthcheck's binary inside that service's pinned image and fails if the
      # runtime cannot find it. It replaces a hand-maintained list of known
      # no-userland images, which could only ever be right about the images
      # somebody had thought to check — so a Dependabot bump onto a distroless
      # base passed it silently (#79).
      #
      # The flag is unconditional here by design: --probe fails rather than
      # skips when there is no daemon, so this step cannot quietly degrade into
      # the static check. scripts/validate.sh may honestly skip it on a host
      # without docker; CI may not.
      # What the step below proves is "binary B execs inside image reference R",
      # and every R in this repository is digest-pinned — so the claim is about
      # immutable bytes and cannot change while the digest does not. Re-proving
      # it on every pull request is what made this job pull ~8 GB from ghcr.io
      # for Markdown-only diffs, and ghcr.io throttled those pulls: anonymously
      # on 2026-09-21, and again on 2026-09-22 WITH a working docker login, where
      # the retry spent its whole 5/20/60 budget and the fourth attempt still
      # failed (#602).
      #
      # So the proof is cached rather than the image. A few KB of digests, not
      # gigabytes of layers — the six ghcr.io images alone are over 6 GB, which
      # would take most of this repository's 10 GB cache budget and still have to
      # be restored on every run.
      #
      # Nothing about the check weakens. --probe stays unconditional and still
      # fails rather than skips without a daemon; this is not a path filter and
      # not a skip. Every target is still accounted for, and any whose proof is
      # not held is pulled and exec'd as before. Only digest-pinned references
      # are cacheable, and the cache carries a hash of check_compose_health.py so
      # editing the prober voids every proof it made.
      #
      # restore-keys is what makes a pin bump cheap rather than total: the exact
      # key misses, the prefix key restores the previous proofs, and only the
      # digest that actually moved is pulled.
      - name: Restore the probe proofs
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .probe-proofs
          key: probe-proofs-${{ hashFiles('stacks/*/compose.yaml', 'scripts/check_compose_health.py') }}
          restore-keys: probe-proofs-

      - name: Verify health dependencies are satisfiable and probe the images
        run: |
          fail=0
          for f in $(./scripts/stacks.sh --paths | sed 's|$|/compose.yaml|'); do
            echo "== $f"
            python3 scripts/check_compose_health.py \
              --probe --proof-cache .probe-proofs/proofs.txt "$f" || fail=1
          done
          # The half no single file can answer: a reloaded or claimed service
          # that exists in no stack at all.
          python3 scripts/check_compose_health.py --cross-stack || fail=1
          exit $fail

      # promtool cannot check these — it parses PromQL and rejects every LogQL
      # stream selector. Loki itself is the only thing that understands them, so
      # the checker boots the pinned image with the rules mounted.
      #
      # It also feeds it the LogQL from every dashboard panel, which is the
      # other half of the "Parse every dashboard PromQL expression" step below:
      # the Prometheus panels went to promtool and the Loki panels went nowhere,
      # so a typo in one rendered an empty panel and looked like quiet traffic
      # rather than a broken query (#82).
      - name: Validate Loki rules and dashboard LogQL
        run: |
          fail=0
          for stack in $(./scripts/stacks.sh); do
            ./scripts/check_loki_rules.sh --stack "$stack" || fail=1
          done
          exit $fail

      # The step above proves the LogQL parses; this proves it matches. Each
      # case pushes fixture lines in the source's real format into the pinned
      # Loki and runs the rule's own expr at a fixed instant, then compares the
      # series that come back (#843). A regex that parses but matches nothing a
      # device writes passed the step above and fails here.
      - name: Behaviour-test the Loki rules
        run: |
          fail=0
          for stack in $(./scripts/stacks.sh); do
            python3 scripts/test_loki_rules.py --stack "$stack" || fail=1
          done
          exit $fail

      # The syslog listener is published off-host, and the Loki security rules
      # read what it stores by {host="morpheus"}. This boots the pinned Alloy
      # and Loki, then sends a spoofed line from an unlisted address and a
      # forged hostname from the allowed one. Neither may change what is stored
      # (#844).
      - name: Check the syslog sender allowlist
        run: ./scripts/check_syslog_senders.sh

      # The Docker socket proxies' HAProxy config: Alloy's three /containers
      # paths answer, and archive, export, top and the rest are refused by the
      # proxy itself (Tecnativa/docker-socket-proxy#182). It also fails when an
      # image bump changes upstream's template under the committed copy.
      - name: Check the socket proxy allowlist
        run: ./scripts/check_socket_proxy.sh

      # scan-images.sh skips gosu in the repositories scripts/gosu-skip.txt
      # lists. This boots each pinned one as 999:999 with a stand-in gosu, and
      # fails if the entrypoint calls it, so an image bump can't turn the skip
      # into hidden findings (#995, #980).
      - name: Check the gosu skip still holds
        run: ./scripts/check_gosu_skip.sh

      # Same shape for the reverse proxy's config: `caddy validate` and
      # `caddy fmt --diff` in the pinned image, per stack, so a Caddyfile that
      # would fail at container start fails here instead (#129). Stacks with
      # no Caddyfile pass through.
      - name: Validate Caddyfiles
        run: |
          fail=0
          for stack in $(./scripts/stacks.sh); do
            ./scripts/check_caddyfile.sh --stack "$stack" || fail=1
          done
          exit $fail

      # The same for Glance's config, in the pinned image (ADR-0098).
      - name: Validate glance.yml
        run: ./scripts/check_glance_config.sh

      # The device list is spread across snmp.yaml, generator.yaml,
      # render-config.sh's REQUIRED array and the example secrets file. Drift
      # between them is invisible until a poll goes out with an empty community,
      # and the generator path fails *open*.
      #
      # This ran in scripts/validate.sh and nowhere else — it was the only check
      # `make validate` had that CI did not, so it had never run on a pull
      # request. That asymmetry is #68.
      # sops silently uses the next rule when one matches nothing, so a
      # recipient separation can exist in the file, be reviewed, be merged, and
      # not be real. That is exactly what happened to the stacks/lab rule
      # ADR-0020 added.
      - name: Verify every .sops.yaml rule matches the file it was written for
        run: python3 scripts/check_sops_rules.py

      - name: Verify the SNMP inventory agrees with itself
        run: ./scripts/snmp-targets.sh --check

      # Documentation was the last defect class here with no check at all. This
      # asserts the prose agrees with the configs it describes — rule and panel
      # counts, SNMP targets against docs/network.md, the host/stack and ports
      # tables against compose.yaml, and image versions quoted in Markdown.
      # #72 (counts stale in eight places) and #73 (six stale image versions,
      # "and CI does not check Markdown") are both this check's absence.
      - name: Verify the documents agree with the configs
        run: python3 scripts/check_docs.py

      # Every url_file alertmanager.yaml reads must be one render-config.sh
      # writes. #214 lost every notification for ten and a half hours because
      # the files were unreadable, and the alert about it travelled the same
      # broken path. This half is pure text — no secret, no host — so it gates a
      # pull request rather than waiting for a deploy to discover it.
      - name: Verify the alert channels are wired end to end
        run: python3 scripts/check_alert_channels.py

      # The schedule's cadence lives in systemd/*.timer and its staleness
      # threshold lives in the JOBS table in install-timers.sh — two copies of
      # one fact, which is what #68 was about. The check that compares them ran
      # in scripts/validate.sh and in no job here, so it guarded a `make
      # validate` a contributor may not run and guarded a pull request not at
      # all. --check now asserts this step exists, so deleting it fails rather
      # than quietly narrowing coverage back.
      #
      # --require-all because a skip in CI is a check nobody runs. The one skip
      # that survives it is `systemd-analyze verify`, which resolves ExecStart
      # against the real filesystem and is meaningless anywhere but the
      # deployment host — escalating that would fail CI for being CI.
      - name: Verify the schedule and its staleness thresholds agree
        run: ./scripts/install-timers.sh --check --require-all

      - name: Validate Grafana dashboards
        run: |
          fail=0
          for stack in $(./scripts/stacks.sh); do
            python3 scripts/check_dashboards.py --stack "$stack" || fail=1
          done
          # A captured dashboard taller than the renderer's ceiling would be
          # cropped by `make screenshots` while it reports success. Fail the PR
          # that grows it, not the monitoring host later.
          ./scripts/capture-screenshots.sh --check || fail=1
          exit $fail

      # Dashboard queries are as easy to typo as alert rules, and a broken one
      # shows up as an empty panel rather than an error. Parse them all — the
      # LogQL half is done by the Loki step above, which is the only thing that
      # can parse it.
      - name: Parse every dashboard PromQL expression
        run: |
          fail=0
          for stack in $(./scripts/stacks.sh); do
            # --emit-promql fails on a stack with no dashboards, deliberately:
            # its output is what proves the panel queries parse, so emitting an
            # empty file would pass promtool over nothing (#68). The guard is
            # here, where "this stack ships none" is knowable.
            if [ -z "$(find "stacks/$stack/grafana/dashboards" -name '*.json' 2>/dev/null)" ]; then
              echo "== $stack: no dashboards"
              continue
            fi
            python3 scripts/check_dashboards.py --stack "$stack" --emit-promql \
              > "/tmp/dashboard-exprs-$stack.yaml"
            docker run --rm --entrypoint promtool \
              -v /tmp:/tmp "$PROM_IMAGE" check rules "/tmp/dashboard-exprs-$stack.yaml" || fail=1
          done
          exit $fail

      # The step above reads the dashboard JSON; this one makes Grafana serve it
      # back. Grafana does not return what it was given — it sorts keys, HTML-
      # escapes `&` and migrates old schema versions — so without this the first
      # real `make dashboards-export` rewrites all seven files and buries the one
      # diff that matters.
      #
      # It also asserts Grafana still accepts a save to a provisioned dashboard.
      # That is the condition the whole export depends on and the one it cannot
      # detect for itself: with allowUiUpdates back at false the command still
      # succeeds, still exits zero, and silently writes nothing (#100).
      - name: Verify the dashboards round-trip through Grafana
        run: ./scripts/check_dashboard_roundtrip.sh

  # ---------------------------------------------------------------------------
  # Home Assistant's hardening — read-only root, every capability dropped,
  # no-new-privileges — was proved by booting it once, against one digest, and
  # Dependabot moves the digest monthly (#534). --probe above execs the
  # healthcheck's binary; it never starts the application. This does: the one
  # service from the real compose.yaml, on a network with no route out, until
  # its own healthcheck passes, and then asks the running container whether the
  # hardening was actually applied.
  #
  # Its own job so it runs beside validate rather than after six minutes of it.
  # The proof is cached on the service's inputs in validate's manner (#602), so
  # a diff that does not touch the service, its config or the check pulls
  # nothing. A bump misses, and boots. Not a path filter, and not a skip.
  hardened-boot:
    name: Boot hardened services
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # validate's login, for validate's reason: the image is on ghcr.io.
      - name: Authenticate to ghcr.io so the pull below is not anonymous
        env:
          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GHCR_USER: ${{ github.actor }}
        run: |
          if printf '%s' "$GHCR_TOKEN" \
            | docker login ghcr.io -u "$GHCR_USER" --password-stdin; then
            echo "ghcr.io: authenticated as $GHCR_USER"
          else
            echo "::warning::ghcr.io login failed — the pull below is anonymous"
          fi

      - name: Restore the boot proofs
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: .boot-proofs
          key: >-
            boot-proofs-${{ hashFiles('stacks/sensitive/compose.yaml',
            'stacks/sensitive/home-assistant/**', 'stacks/sensitive/ntfy/**',
            'stacks/sensitive/stirling-pdf/**', 'stacks/sensitive/mosquitto/**',
            'stacks/sensitive/ring-mqtt/**', 'stacks/soc/compose.yaml',
            'scripts/seed-validation-env.sh', 'scripts/check_hardened_boot.sh') }}
          restore-keys: boot-proofs-

      - name: Boot Home Assistant under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service home-assistant --proof-cache .boot-proofs/proofs.txt

      # ntfy exits at start on a malformed hash or token, which is why
      # seed-validation-env.sh seeds it well-formed ones (#136); this is the
      # step that would say so if it stopped.
      - name: Boot ntfy under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service ntfy --proof-cache .boot-proofs/proofs.txt

      # linkding's bootstrap runs as root on four capabilities and drops to
      # 33 (ADR-0061); an image that moved its data directory or its user
      # would stop booting here rather than on trinity.
      - name: Boot linkding under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service linkding --proof-cache .boot-proofs/proofs.txt
      # Stirling-PDF runs read-only as a non-root uid its entrypoint was
      # written to drop to, not start as, with every document on tmpfs
      # (ADR-0063). Each of those is a way for an upstream bump to stop it
      # booting, and this is where that would show. Its healthcheck can pass
      # while pdfium cannot load, so stirling-pdf/smoke.sh merges two pages
      # after the boot as well.
      - name: Boot Stirling-PDF under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service stirling-pdf --proof-cache .boot-proofs/proofs.txt

      # Actual runs as the image's own user with a read-only root and no
      # tmpfs of its own: every one of those is a way a node app can fail at
      # start, and an image that begins writing outside /data would fail here
      # first rather than on trinity (#142, ADR-0062).
      - name: Boot Actual under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service actual --proof-cache .boot-proofs/proofs.txt

      # The Ring Alarm's broker and bridge (ADR-0104). Mosquitto builds its
      # password file at start and runs read-only as 1883, and its smoke.sh
      # signs in as both users and checks the ACL. ring-mqtt runs as plain
      # Node, not its image's s6 init, as RENDER_UID on a read-only root;
      # its smoke.sh checks that it read its config and waits for a token.
      - name: Boot Mosquitto under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service mosquitto --proof-cache .boot-proofs/proofs.txt

      - name: Boot ring-mqtt under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack sensitive
          --service ring-mqtt --proof-cache .boot-proofs/proofs.txt

      # The SOC stack's MSI server (#1068) is a numeric user, read-only, with
      # every capability dropped but NET_BIND_SERVICE. Its first deploy on odin
      # crash-looped on "exec caddy failed: Operation not permitted": the
      # image's binary carries that capability as a file capability, and
      # #1071's compose dropped it. Nothing here booted it, so odin was the
      # first place that showed. This is where a Caddy bump, or a cap_add
      # tidied away, would show next. Its Caddyfile is Validate configs' (the
      # Caddyfile check); this proves the container starts and is healthy.
      - name: Boot the SOC MSI server under its hardening
        run: >-
          ./scripts/check_hardened_boot.sh --stack soc
          --service caddy --proof-cache .boot-proofs/proofs.txt

  # ---------------------------------------------------------------------------
  secret-scan:
    name: Secret scan
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          fetch-depth: 0

      # Same single source of truth as the validate job: whatever compose.yaml
      # pins is what runs. Repeated here rather than shared because jobs do not
      # share $GITHUB_ENV — it is the resolution that is duplicated, not the
      # version.
      - name: Resolve the pinned gitleaks image from compose.yaml
        run: |
          echo "GITLEAKS_IMAGE=$(./scripts/image-for.sh gitleaks)" >> "$GITHUB_ENV"
          ./scripts/image-for.sh gitleaks

      # Two scans, because they answer different questions.
      #
      # The working-tree scan asks "is there a secret in the code as it stands"
      # and must always be clean. The history scan asks "has one ever been
      # committed", and must now be clean unconditionally too: there is no
      # .gitleaksignore any more. The purge in docs/runbooks/purge-git-history.md
      # removed what that file used to acknowledge, and deleting it is how a
      # future finding stays visible instead of being masked by a stale entry.
      #
      # Two configurations, because a path allowlist applies to every commit a
      # history scan walks. The working tree reads .gitleaks-worktree.toml,
      # which allowlists the decrypted and generated paths a deployed host
      # holds; the history scan reads .gitleaks.toml, which allowlists none of
      # them, so a purged key that reappears under certificates/ is a finding.
      #
      # Both run with --network none, as lint.sh's linters do: gitleaks reads
      # the checkout and a local `git log`, and on a fork PR that checkout is
      # the author's. For the same reason the checkout is mounted read-only and
      # gitleaks, whose image runs as root, runs as the runner's user. A
      # gitleaks that the checkout subverts must not be able to rewrite
      # scripts/gitleaks-history.sh, which the next step runs on the runner.
      - name: gitleaks — working tree
        run: |
          docker run --rm --network none --user "$(id -u):$(id -g)" -e HOME=/tmp \
            -v "$PWD:/repo:ro" -w /repo \
            "$GITLEAKS_IMAGE" \
            detect --no-git --no-banner --redact -c .gitleaks-worktree.toml -v

      # Through the script validate.sh also calls. gitleaks exits 0 when git
      # cannot read the repository, so the script also fails a scan of zero
      # commits. The full clone above makes that impossible today. The check is
      # there for the day the checkout changes.
      - name: gitleaks — full history
        run: |
          ./scripts/gitleaks-history.sh docker run --rm --network none \
            --user "$(id -u):$(id -g)" -e HOME=/tmp -v "$PWD:/repo:ro" -w /repo \
            "$GITLEAKS_IMAGE"

      # gitleaks cannot be the control here. .purge-secrets.txt is gitignored so
      # the filesystem scan skips it, and its contents are bare literals with no
      # keyword context for a rule to match. The control is simply that it must
      # never be a tracked file.
      # Both of these were inline here and unrunnable locally, which is the
      # wrong place for a check whose whole value is catching a plaintext
      # secret BEFORE it reaches the remote (#175). The tracked-artefact one
      # was also written out a second time in validate.sh, and the two copies
      # had drifted. One script each, run from both.
      - name: Assert no decrypted artefact is tracked
        run: ./scripts/check-tracked-artefacts.sh

      # The one .pem the repository tracks (#764) must stay a CA certificate.
      # CI has no certificates/ca.pem, so the host-copy comparison is the
      # monitoring host's validate.sh; this run proves the shape.
      - name: Assert the committed ingest CA is a CA certificate only
        run: ./scripts/check_ingest_ca.sh --self-test && ./scripts/check_ingest_ca.sh

      - name: Assert SOPS files are actually encrypted
        run: ./scripts/check-sops-encrypted.sh