diff --git a/.config/hakari.toml b/.config/hakari.toml index f3ae2869052..1c32ee32ab5 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -55,4 +55,8 @@ third-party = [ # a feature only we can use that; until then, it's fine to just exclude # this crate entirely: it's not very big and isn't used by many things. { name = "scuffle" }, + + # TODO-RAINCLAUDE: the Antithesis SDK must stay a no-op except under the `antithesis` features of omicron-nexus and omicron-antithesis-workload; unifying it through the workspace-hack would turn on `full` for every build. + { name = "antithesis_sdk" }, + { name = "antithesis-instrumentation" }, ] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..9ee943c7218 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +# TODO-RAINCLAUDE: keeps build outputs and VCS state out of the Antithesis image build context (antithesis/Dockerfile). +target/ +out/ +.jj/ +.git/ diff --git a/.snouty.toml b/.snouty.toml new file mode 100644 index 00000000000..ec9ea89e6c0 --- /dev/null +++ b/.snouty.toml @@ -0,0 +1,5 @@ +# TODO-RAINCLAUDE: project-level snouty settings (see antithesis/README.adoc). Credentials never go here: snouty reads the API key from ANTITHESIS_API_KEY or its credential store. `container_engine` is set because this repo's developers use rootless podman with Compose v2. +tenant = "oxide" +# TODO-RAINCLAUDE: the full registry path, not the bare repository name: snouty pushes to `{repository}/{image}:{tag}`, and a bare name sends podman to docker.io. +repository = "us-central1-docker.pkg.dev/molten-verve-216720/oxide-repository" +container_engine = "podman" diff --git a/Cargo.lock b/Cargo.lock index 4c4bae37bb4..b7eef50c186 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "antithesis-instrumentation" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb6b548668212c6d3a942a4ac7be13bc4fc25715bf661328438f30e3fd342cf0" +dependencies = [ + "cc", +] + +[[package]] +name = "antithesis_sdk" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08410fcac93669a476c006cd6c4512ac1e2b30fd117231a5d55d8a2c76599b82" +dependencies = [ + "libc", + "libloading", + "linkme", + "once_cell", + "rand 0.8.6", + "rustc_version_runtime", + "serde", + "serde_json", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -3153,6 +3178,7 @@ dependencies = [ name = "dns-server" version = "0.1.0" dependencies = [ + "antithesis-instrumentation", "anyhow", "camino", "camino-tempfile", @@ -7438,6 +7464,7 @@ dependencies = [ name = "nexus-db-queries" version = "0.1.0" dependencies = [ + "antithesis_sdk", "anyhow", "assert_matches", "async-bb8-diesel", @@ -8745,6 +8772,28 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "omicron-antithesis-workload" +version = "0.1.0" +dependencies = [ + "antithesis-instrumentation", + "antithesis_sdk", + "anyhow", + "clap", + "dropshot", + "futures", + "http", + "nexus-config", + "nexus-lockstep-client", + "omicron-workspace-hack", + "oxide-client", + "oxide-tokio-rt", + "reqwest 0.13.2", + "serde_json", + "slog", + "tokio", +] + [[package]] name = "omicron-certificates" version = "0.1.0" @@ -9198,6 +9247,8 @@ dependencies = [ name = "omicron-nexus" version = "0.1.0" dependencies = [ + "antithesis-instrumentation", + "antithesis_sdk", "anyhow", "assert_matches", "async-bb8-diesel", @@ -9427,6 +9478,7 @@ dependencies = [ name = "omicron-omdb" version = "0.1.0" dependencies = [ + "antithesis-instrumentation", "anyhow", "async-bb8-diesel", "async-trait", @@ -9716,6 +9768,7 @@ dependencies = [ name = "omicron-sled-agent" version = "0.1.0" dependencies = [ + "antithesis-instrumentation", "anyhow", "assert_matches", "async-trait", @@ -9737,6 +9790,7 @@ dependencies = [ "derive_more 0.99.20", "dice-verifier 0.3.0-pre0 (git+https://github.com/oxidecomputer/dice-util?branch=main)", "display-error-chain", + "dns-service-client", "dpd-client 0.1.0 (git+https://github.com/oxidecomputer/dendrite?rev=cc0c307c617f2988aafdca4e3bd35ea178b64801)", "dropshot", "expectorate", @@ -10020,9 +10074,7 @@ dependencies = [ "predicates", "proc-macro2", "quote", - "rand 0.8.6", "rand 0.9.2", - "rand_chacha 0.3.1", "rand_chacha 0.9.0", "regex", "regex-automata", @@ -13158,6 +13210,16 @@ dependencies = [ "semver 1.0.28", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version 0.4.1", + "semver 1.0.28", +] + [[package]] name = "rustix" version = "0.38.44" diff --git a/Cargo.toml b/Cargo.toml index f48f6f45e5b..f3b630ca73a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "antithesis/workload", "api_identity", "bootstore", "certificates", @@ -195,6 +196,7 @@ members = [ ] default-members = [ + "antithesis/workload", "api_identity", "bootstore", "certificates", @@ -439,6 +441,8 @@ cast_lossless = "warn" [workspace.dependencies] anyhow = "1.0" anstyle = "1.0.11" +antithesis-instrumentation = "0.1.0" +antithesis_sdk = { version = "0.2.9", default-features = false } api_identity = { path = "api_identity" } approx = "0.5.1" assert_matches = "1.5.0" @@ -988,6 +992,12 @@ opt-level = 3 [profile.release] panic = "abort" +# TODO-RAINCLAUDE: profile for Antithesis binaries (antithesis/README.adoc); keeps DWARF line tables unstripped for symbolization, while sancov instrumentation comes from antithesis/cargo-config.toml. +[profile.antithesis] +inherits = "release" +debug = "line-tables-only" +strip = "none" + # proptest based test generation and shrinking is expensive. Let's optimize it. [profile.dev.package.proptest] opt-level = 3 diff --git a/antithesis/Dockerfile b/antithesis/Dockerfile new file mode 100644 index 00000000000..506db909e51 --- /dev/null +++ b/antithesis/Dockerfile @@ -0,0 +1,120 @@ +# syntax=docker/dockerfile:1 +# TODO-RAINCLAUDE: images for running the simulated control plane under Antithesis; build via antithesis/config/docker-compose.yaml (context is the repo root). + +# TODO-RAINCLAUDE: the toolchain here must match rust-toolchain.toml (rustup would otherwise download a second one), and the Debian release must match the runtime images below so the binaries link against the same libc and xmlsec generation they run on. Debian releases are pinned rather than `stable` because package names change across releases (trixie renamed libxmlsec1 to libxmlsec1t64). +FROM rust:1.97.1-trixie AS builder-base + +# TODO-RAINCLAUDE: package list mirrors the Linux branch of tools/install_builder_prerequisites.sh. +RUN apt-get update && apt-get install -y --no-install-recommends \ + libclang-dev \ + libpq-dev \ + libxmlsec1-dev \ + libxmlsec1-openssl \ + pkg-config \ + xmlsec1 \ + && rm -rf /var/lib/apt/lists/* + +# TODO-RAINCLAUDE: build scripts materialize `.gitstub` files through git-stub-vcs, which needs real history at the repo root and rejects shallow clones. The build context carries no VCS (see .dockerignore), so the builder gets a treeless partial clone: commits only, with trees and blobs fetched on demand, which git-stub-vcs does not classify as shallow. It must not be bare because cargo walks the package's git repo to fingerprint build scripts and rejects bare repos; --no-checkout leaves the index empty so cargo sees every source file as untracked and lists it. Cloned before COPY so the layer survives source edits; the fetch after COPY picks up commits newer than the cached clone. +ARG OMICRON_GIT_URL=https://github.com/oxidecomputer/omicron +RUN git clone --no-checkout --filter=tree:0 "$OMICRON_GIT_URL" /omicron-history + +WORKDIR /omicron +COPY . . +RUN git -C /omicron-history fetch --filter=tree:0 origin \ + && printf 'gitdir: /omicron-history/.git\n' > /omicron/.git + + +# TODO-RAINCLAUDE: no cargo here on purpose; compose builds images in parallel and two cargo processes racing on the shared registry cache mount corrupt it. The pin and checksum are the same ones cargo xtask download uses (tools/cockroachdb_checksums), and the URL mirrors dev-tools/downloader/src/lib.rs. +FROM debian:trixie-slim AS cockroach-download + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY tools/cockroachdb_checksums /cockroachdb_checksums + +RUN set -eu \ + && . /cockroachdb_checksums \ + && curl --fail --silent --show-error --location \ + --output /cockroach.tgz \ + "https://buildomat.eng.oxide.computer/public/file/oxidecomputer/cockroach/linux-amd64/${COCKROACH_COMMIT}/cockroach.tgz" \ + && echo "${CIDL_SHA256_LINUX} /cockroach.tgz" | sha256sum --check --strict \ + && mkdir -p /out /unpack \ + && tar -xzf /cockroach.tgz -C /unpack \ + && cp /unpack/cockroach/cockroach /out/cockroach \ + && /out/cockroach version + + +FROM builder-base AS builder + +# TODO-RAINCLAUDE: the sancov rustflags in antithesis/cargo-config.toml plus the `antithesis` features are what make these binaries Antithesis-instrumented; sharing=locked keeps any future concurrent cargo stage from racing on these caches. +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/omicron/target,sharing=locked \ + mkdir -p /out \ + && cargo build \ + --profile antithesis \ + --config antithesis/cargo-config.toml \ + --features omicron-nexus/antithesis,omicron-antithesis-workload/antithesis,dns-server/antithesis,omicron-sled-agent/antithesis,omicron-omdb/antithesis \ + --bin nexus \ + --bin sled-agent-sim \ + --bin dns-server \ + --bin schema-updater \ + --bin omdb \ + --bin omicron-antithesis-workload \ + && for bin in nexus sled-agent-sim dns-server schema-updater omdb omicron-antithesis-workload; do \ + cp "target/x86_64-unknown-linux-gnu/antithesis/$bin" "/out/$bin"; \ + done + + +FROM debian:trixie-slim AS omicron-antithesis + +# TODO-RAINCLAUDE: libpq5 is the runtime half of the pq-sys dependency and libxmlsec1t64-openssl (with the libxmlsec1t64 and libxml2 it pulls in) is the runtime half of samael's SAML support; curl is for compose healthchecks. Verify with `ldd /opt/oxide/bin/* | grep "not found"` after changing the builder image. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libpq5 \ + libxmlsec1t64-openssl \ + && rm -rf /var/lib/apt/lists/* + +ENV NO_COLOR=1 +ENV PATH=/opt/oxide/bin:$PATH + +COPY --from=builder /out/nexus /out/sled-agent-sim /out/dns-server /out/schema-updater /out/omdb /out/omicron-antithesis-workload /opt/oxide/bin/ +COPY schema/crdb /opt/oxide/sc hema/crdb +# TODO-RAINCLAUDE: service configs are baked in rather than bind-mounted from antithesis/config so the image runs identically under Antithesis and under rootless podman on SELinux hosts, where host file mounts are unreadable without relabeling. +COPY antithesis/config/nexus.toml antithesis/config/dns-server.toml /opt/oxide/config/ + +# TODO-RAINCLAUDE: Antithesis reads DWARF from /symbols; the binaries are unstripped so symlinks are enough. +RUN mkdir -p /symbols /var/oxide/dns-storage /var/tmp/omicron_tmp \ + && for bin in nexus sled-agent-sim dns-server schema-updater omdb omicron-antithesis-workload; do \ + ln -s "/opt/oxide/bin/$bin" "/symbols/$bin"; \ + done + + +# TODO-RAINCLAUDE: the test commands live only in the workload image; Antithesis discovers /opt/antithesis/test/v1 in every container and would otherwise run each command once per service built from the shared image. +FROM omicron-antithesis AS omicron-antithesis-workload + +COPY antithesis/test/v1 /opt/antithesis/test/v1 + + +FROM debian:trixie-slim AS omicron-antithesis-cockroach + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + +ENV NO_COLOR=1 +ENV PATH=/opt/oxide/bin:$PATH + +COPY --from=cockroach-download /out/cockroach /opt/oxide/bin/cockroach +COPY schema/crdb/dbinit.sql /opt/oxide/schema/crdb/dbinit.sql +COPY antithesis/cockroach-seed-antithesis.sql /opt/oxide/schema/cockroach-seed-antithesis.sql +COPY antithesis/cockroach-healthcheck.sh antithesis/cockroach-seed.sh /opt/oxide/bin/ + +# TODO-RAINCLAUDE: seeding at build time keeps the runtime hermetic and fast; the store already holds the omicron schema at the version Nexus expects. +RUN /opt/oxide/bin/cockroach-seed.sh /data + +VOLUME /data diff --git a/antithesis/README.adoc b/antithesis/README.adoc new file mode 100644 index 00000000000..ec9eb7783e3 --- /dev/null +++ b/antithesis/README.adoc @@ -0,0 +1,230 @@ +:showtitle: +:toc: left +:icons: font + += Running Omicron under Antithesis + +This directory holds everything needed to run the simulated control plane +(Nexus, CockroachDB, internal DNS, and a simulated sled agent) inside +https://antithesis.com[Antithesis], whose fault injector partitions, throttles, +pauses, and (when enabled) kills containers while properties asserted through +the Antithesis SDK are checked. + +The plan behind this layout, including the phased roadmap and the seed property +catalog, is in `scratchbook/plan.md`. The layout follows what the +https://github.com/antithesishq/antithesis-skills[antithesis-skills] expect, so +those skills and the `snouty` CLI work here unmodified. + +== Layout + +[cols="1,3"] +|=== +| Path | Purpose + +| `Dockerfile` +| Multi-stage build. `omicron-antithesis` holds the instrumented Rust binaries + (`nexus`, `sled-agent-sim`, `dns-server`, `schema-updater`, `omdb`, + `omicron-antithesis-workload`), the CockroachDB schema, and the test commands. + `omicron-antithesis-cockroach` holds the pinned Oxide CockroachDB fork with a + store pre-seeded from `schema/crdb/dbinit.sql`. + +| `cargo-config.toml` +| The rustflags that add Antithesis coverage instrumentation. Passed + explicitly with `cargo build --config`; never part of `.cargo/config.toml`. + +| `config/docker-compose.yaml` +| The five-container topology on an IPv6 network. The addresses in it, in + `config/nexus.toml`, and in the `sled-agent-sim` arguments must agree. + +| `config/nexus.toml`, `config/dns-server.toml` +| Service configuration mounted into the containers. + +| `test/v1/nexus/` +| Antithesis test commands. The filename prefix (`first_`, `parallel_driver_`, + `eventually_`, and so on) tells Antithesis when to run each one. They are + baked into the `omicron-antithesis` image at `/opt/antithesis/test/v1`. + +| `workload/` +| The `omicron-antithesis-workload` crate that the test commands call. It + emits the `setup_complete` signal, seeds a project and IP pool, and checks + properties. + +| `scratchbook/` +| Analysis artifacts consumed by the `antithesis-research`, `antithesis-setup`, + and `antithesis-workload` skills. +|=== + +== How the pieces fit together + +. `cockroach` starts on a pre-seeded store, so no migration runs at boot. +. `internal-dns` starts empty. +. `nexus` starts its internal and lockstep APIs and waits for rack + initialization. Its healthcheck only checks the internal API; the external + API does not exist until the handoff below. +. `sled-agent-sim` registers a simulated Gimlet with Nexus, populates + `internal-dns` (including the CockroachDB record), and performs the rack + setup handoff. This is the same path `omicron-dev run-all` exercises, but + across container boundaries. +. `workload` runs `omicron-antithesis-workload wait-ready`, which waits for + the external API, a successful login as the recovery user, and a registered + sled, then writes the Antithesis `setup_complete` signal. Testing starts + from that moment. + +The recovery silo is `demo-silo` and the user is `demo-privileged` with +password `oxide`; both names are `sled-agent-sim` options. + +== Building the images + +The build runs on x86-64 Linux only (Antithesis runs x86-64 containers, and the +instrumentation flags target `x86_64-unknown-linux-gnu`). From the repository +root: + +[source,text] +---- +$ docker compose -f antithesis/config/docker-compose.yaml build +---- + +Docker is not required. Rootless podman works as long as `podman compose` +resolves to Compose v2 rather than podman-compose (install the +`docker-compose` binary from https://github.com/docker/compose/releases into +`~/.docker/cli-plugins/`); then substitute `podman` for `docker` in every +command in this document. Paths handed to podman must be visible to the host +that runs it. + +The build context deliberately excludes `.git` and `.jj` (see the repository's +`.dockerignore`). Build scripts that materialize `.gitstub` files still need +git history, so the builder stage makes a treeless, checkout-less partial clone of +`https://github.com/oxidecomputer/omicron` (override with +`--build-arg OMICRON_GIT_URL=...`) and points `/omicron/.git` at it. A stub +that references a commit not yet pushed to that remote will fail to +materialize inside the image build. + +The builder stage compiles the workspace once with `--profile antithesis` and +the flags from `cargo-config.toml`. Expect this to take well over an hour on a +laptop the first time; the `target` and registry directories are BuildKit +cache mounts, so rebuilds are incremental. + +To check that a binary is instrumented: + +[source,text] +---- +$ docker run --rm omicron-antithesis:latest sh -c 'nm /opt/oxide/bin/nexus | grep antithesis_load_libvoidstar' +$ docker run --rm omicron-antithesis:latest sh -c 'readelf -S /opt/oxide/bin/nexus | grep sancov' +---- + +== Running locally + +[source,text] +---- +$ docker compose -f antithesis/config/docker-compose.yaml up +---- + +Wait for the `workload` container to log `sent setup_complete`. Then the test +commands can be exercised by hand: + +[source,text] +---- +$ docker compose -f antithesis/config/docker-compose.yaml exec workload /opt/antithesis/test/v1/nexus/first_seed.sh +$ docker compose -f antithesis/config/docker-compose.yaml exec workload /opt/antithesis/test/v1/nexus/eventually_sagas_settle.sh +---- + +The external API is reachable from the host only if Docker publishes it; +inside the network it is `http://[fd00:1122:3344:101::5]:12220`. + +=== Proving the deployment is hermetic + +Antithesis runs without internet access. To confirm nothing at runtime depends +on it, bring the stack up inside a network namespace with no external +connectivity: + +[source,text] +---- +$ sudo unshare -n docker compose -f antithesis/config/docker-compose.yaml up +---- + +=== Validating with snouty + +With https://github.com/antithesishq/snouty[snouty] installed: + +[source,text] +---- +$ snouty doctor +$ SNOUTY_TEMP_DIR=$HOME/.cache/snouty-validate-$(date +%s) snouty validate antithesis/config --timeout 480 +---- + +`validate` needs no credentials at all; `doctor` needs them only for its API +connectivity check (see <>). `--timeout` is in seconds +(the default of 120 is enough here; the stack reaches `setup_complete` +within about a minute). `SNOUTY_TEMP_DIR` must be a fresh directory that the +host's container engine can bind-mount: snouty relabels it for SELinux and +watches it for the `setup_complete` event, and the repository's +`.snouty.toml` selects podman. A successful run ends with +`Setup validation successful.` and a count of discovered test commands; the +count should match the files in `test/v1/nexus/` exactly once, because only +the workload image carries them. + +`snouty validate` builds the images, brings the stack up, and waits for the +`setup_complete` signal. + +[#Authenticating] +=== Authenticating + +The tenant (`oxide`) and repository +(`us-central1-docker.pkg.dev/molten-verve-216720/oxide-repository`) are in +`.snouty.toml` at the repository root. The repository has to be that whole +path: `snouty` pushes to `{repository}/{image}:{tag}`, so a bare +`oxide-repository` sends the push to Docker Hub, which fails late with a +confusing `requested access to the resource is denied` after uploading +several gigabytes of blobs. Two secrets are not in the file: an API key, which +`snouty` uses to talk to the API, and a Google service-account key, which the +container engine uses to push images to Antithesis's registry. Oxide keeps +both in 1Password. + +`snouty` reads the API key from the environment, so it never has to be stored +on disk: + +[source,text] +---- +$ export ANTITHESIS_API_KEY="$(op read 'op://Private/antithesis api key/notesPlain' | tail -1)" +$ snouty doctor +---- + +`snouty login` is the alternative; it persists the key under `$HOME` instead. + +Pushing images needs a separate `docker login` against Google Artifact +Registry, which `snouty launch` then reuses: + +[source,text] +---- +$ op read 'op://Private/antithesis gar key/notesPlain' \ + | docker login -u _json_key --password-stdin us-central1-docker.pkg.dev +---- + +Two notes for podman users. Drop the `https://` scheme Antithesis's own +instructions carry: podman rejects it. And the credential lands in +`$XDG_RUNTIME_DIR/containers/auth.json`, which is a tmpfs, so the login has to +be repeated after a reboot. If podman runs on the host through a distrobox +shim, run this so that it reaches the same host engine that builds and pushes. + +== Adding a property + +Assertions inside Nexus use `antithesis_sdk` directly (see +`nexus/src/app/saga.rs` and `nexus/db-queries/src/transaction_retry.rs`). The +crate is a no-op unless the `antithesis` feature of `omicron-nexus` is enabled, +which only the Dockerfile does, so nothing changes for production builds or +the test suite. Assertion names must be string literals: Antithesis catalogs +them statically. + +Properties checked from outside Nexus go in the workload crate, with a +wrapper script in `test/v1/nexus/`. Scripts whose names start with `helper_` +are ignored by Antithesis. + +== Still blocked on Antithesis + +* Confirmation that Antithesis's compose environment supports IPv6 networks + with static `ipv6_address` assignments. Omicron's internal DNS records are + IPv6-only, so this topology depends on it. The fallback in + `scratchbook/plan.md` keeps `cockroach` and `workload` as separate + containers over IPv4 and co-locates the rest on `::1`. +* Enabling node termination faults for the `nexus` container once saga + recovery properties exist. diff --git a/antithesis/cargo-config.toml b/antithesis/cargo-config.toml new file mode 100644 index 00000000000..332c4ea0c2c --- /dev/null +++ b/antithesis/cargo-config.toml @@ -0,0 +1,15 @@ +# TODO-RAINCLAUDE: pass with `cargo build --config antithesis/cargo-config.toml`; kept out of .cargo/config.toml because sancov slows every crate's build. +# TODO-RAINCLAUDE: an explicit build.target keeps the rustflags off host artifacts (build scripts, proc macros), which cannot link without the instrumentation crate's sancov hooks. +[build] +target = "x86_64-unknown-linux-gnu" + +# TODO-RAINCLAUDE: target rustflags replace build.rustflags, so --cfg tokio_unstable is repeated here; --build-id is required by Antithesis for symbolization. +[target.x86_64-unknown-linux-gnu] +rustflags = [ + "--cfg", "tokio_unstable", + "-Ccodegen-units=1", + "-Cpasses=sancov-module", + "-Cllvm-args=-sanitizer-coverage-level=3", + "-Cllvm-args=-sanitizer-coverage-trace-pc-guard", + "-Clink-args=-Wl,--build-id", +] diff --git a/antithesis/cockroach-healthcheck.sh b/antithesis/cockroach-healthcheck.sh new file mode 100755 index 00000000000..94b36a7571b --- /dev/null +++ b/antithesis/cockroach-healthcheck.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# TODO-RAINCLAUDE: compose healthcheck; ready=1 makes CockroachDB report unhealthy while draining or before it accepts SQL. + +set -euo pipefail + +exec curl --silent --fail --output /dev/null "http://127.0.0.1:8080/health?ready=1" diff --git a/antithesis/cockroach-seed-antithesis.sql b/antithesis/cockroach-seed-antithesis.sql new file mode 100644 index 00000000000..bbf861e3834 --- /dev/null +++ b/antithesis/cockroach-seed-antithesis.sql @@ -0,0 +1,5 @@ +-- TODO-RAINCLAUDE: applied by cockroach-seed.sh after dbinit.sql. Rows here stand in for state that real deployments get from services the phase 1 Antithesis topology does not run. + +-- TODO-RAINCLAUDE: the RSS handoff sets a port-settings id on the uplink port named in the rack network config (nexus/src/app/rack.rs, `switch_port_get_id`), and that row is normally created by Nexus's populate_switch_ports task from dendrite's port list. There is no dendrite here, so the placeholder uplink port that sled-agent-sim reports is created up front. The rack id is the one hard-coded in sled-agent/src/sim/server.rs `handoff_to_nexus` and in antithesis/config/nexus.toml. +INSERT INTO omicron.public.switch_port (id, rack_id, port_name, port_settings_id, switch_slot) +VALUES (gen_random_uuid(), 'c19a698f-c6f9-4a17-ae30-20d711b8f7dc', 'qsfp0', NULL, 'switch0'); diff --git a/antithesis/cockroach-seed.sh b/antithesis/cockroach-seed.sh new file mode 100755 index 00000000000..1f8fdf3ddde --- /dev/null +++ b/antithesis/cockroach-seed.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# TODO-RAINCLAUDE: populates a fresh single-node CockroachDB store with schema/crdb/dbinit.sql, then stops the node cleanly so the image ships a consistent store. + +set -euo pipefail + +store_dir="${1:?usage: cockroach-seed.sh STORE_DIR}" +listen_addr="127.0.0.1:26257" +http_addr="127.0.0.1:8080" +pid_file="/tmp/cockroach-seed.pid" + +cockroach start-single-node \ + --insecure \ + --store="path=${store_dir},ballast-size=0" \ + --listen-addr="${listen_addr}" \ + --http-addr="${http_addr}" \ + --pid-file="${pid_file}" \ + --background + +cockroach sql \ + --insecure \ + --host="${listen_addr}" \ + --file=/opt/oxide/schema/crdb/dbinit.sql + +cockroach sql \ + --insecure \ + --host="${listen_addr}" \ + --file=/opt/oxide/schema/cockroach-seed-antithesis.sql + +version="$(cockroach sql --insecure --host="${listen_addr}" --format=csv \ + --execute='SELECT version FROM omicron.public.db_metadata' | tail -n 1)" +echo "seeded omicron database at schema version ${version}" + +cockroach node drain --insecure --host="${listen_addr}" --self +kill -TERM "$(cat "${pid_file}")" +while kill -0 "$(cat "${pid_file}")" 2>/dev/null; do + sleep 1 +done +rm -f "${pid_file}" diff --git a/antithesis/config/dns-server.toml b/antithesis/config/dns-server.toml new file mode 100644 index 00000000000..68c53cff3c1 --- /dev/null +++ b/antithesis/config/dns-server.toml @@ -0,0 +1,10 @@ +[dropshot] +default_request_body_max_bytes = 104857600 + +[log] +level = "info" +mode = "stderr-terminal" + +[storage] +storage_path = "/var/oxide/dns-storage" +keep_old_generations = 3 diff --git a/antithesis/config/docker-compose.yaml b/antithesis/config/docker-compose.yaml new file mode 100644 index 00000000000..eea345b7ac0 --- /dev/null +++ b/antithesis/config/docker-compose.yaml @@ -0,0 +1,150 @@ +# TODO-RAINCLAUDE: Antithesis deployment of the simulated control plane; addresses here must match antithesis/config/nexus.toml and the sled-agent-sim arguments. +name: omicron + +networks: + underlay: + enable_ipv6: true + ipam: + config: + - subnet: fd00:1122:3344:101::/64 + +services: + cockroach: + container_name: cockroach + hostname: cockroach + platform: linux/amd64 + init: true + build: + context: ../.. + dockerfile: antithesis/Dockerfile + target: omicron-antithesis-cockroach + image: omicron-antithesis-cockroach:latest + environment: + NO_COLOR: "1" + # TODO-RAINCLAUDE: same flags as test-utils/src/dev/db.rs apart from the fixed addresses. + command: + - cockroach + - start-single-node + - --insecure + - --store=path=/data,ballast-size=0 + - --listen-addr=[::]:32221 + - --http-addr=[::]:8080 + - --max-sql-memory=256MiB + healthcheck: + test: ["CMD", "/opt/oxide/bin/cockroach-healthcheck.sh"] + interval: 2s + timeout: 2s + retries: 60 + networks: + underlay: + ipv6_address: fd00:1122:3344:101::4 + + # TODO-RAINCLAUDE: only the nexus service carries the build: for omicron-antithesis; podman's compose path builds once per service that declares build:, so the other users of the image reference it by name. + internal-dns: + container_name: internal-dns + hostname: internal-dns + platform: linux/amd64 + init: true + image: omicron-antithesis:latest + environment: + NO_COLOR: "1" + command: + - dns-server + - --config-file=/opt/oxide/config/dns-server.toml + - --http-address=[fd00:1122:3344:101::3]:5353 + - --dns-address=[fd00:1122:3344:101::3]:53 + # TODO-RAINCLAUDE: a plain TCP connect rather than an HTTP request: the dns-server API is versioned and logs an error for every request without an api-version header, and pinning a version here would rot. + healthcheck: + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/fd00:1122:3344:101::3/5353"] + interval: 2s + timeout: 2s + retries: 30 + networks: + underlay: + ipv6_address: fd00:1122:3344:101::3 + + nexus: + container_name: nexus + hostname: nexus + platform: linux/amd64 + init: true + build: + context: ../.. + dockerfile: antithesis/Dockerfile + target: omicron-antithesis + image: omicron-antithesis:latest + environment: + NO_COLOR: "1" + command: + - nexus + - /opt/oxide/config/nexus.toml + depends_on: + cockroach: + condition: service_healthy + internal-dns: + condition: service_healthy + # TODO-RAINCLAUDE: healthy means the internal API accepts TCP connections; the external API only appears after the sled-agent-sim handoff, so checking it here would deadlock, and an HTTP request without an api-version header is logged as an error on every probe. + healthcheck: + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/fd00:1122:3344:101::5/12221"] + interval: 2s + timeout: 2s + retries: 60 + networks: + underlay: + ipv6_address: fd00:1122:3344:101::5 + + sled-agent-sim: + container_name: sled-agent-sim + hostname: sled-agent-sim + platform: linux/amd64 + init: true + image: omicron-antithesis:latest + environment: + NO_COLOR: "1" + command: + - sled-agent-sim + - b6d65341-167c-41df-9b5c-41cded99c229 + - "[fd00:1122:3344:101::6]:12345" + - "[fd00:1122:3344:101::5]:12221" + - "12232" + - --rss-nexus-external-addr=[fd00:1122:3344:101::5]:12220 + # TODO-RAINCLAUDE: deployment.id in nexus.toml. The recovery silo and user are left at sled-agent-sim's defaults (demo-silo, demo-privileged, password oxide), which the workload also defaults to. + - --rss-nexus-id=e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c + - --rss-internal-dns-dns-addr=[fd00:1122:3344:101::3]:53 + - --rss-internal-dns-http-addr=[fd00:1122:3344:101::3]:5353 + - --cockroach-addr=[fd00:1122:3344:101::4]:32221 + - --rack-subnet=fd00:1122:3344::/56 + depends_on: + nexus: + condition: service_healthy + internal-dns: + condition: service_healthy + networks: + underlay: + ipv6_address: fd00:1122:3344:101::6 + + workload: + container_name: workload + hostname: workload + platform: linux/amd64 + init: true + build: + context: ../.. + dockerfile: antithesis/Dockerfile + target: omicron-antithesis-workload + image: omicron-antithesis-workload:latest + environment: + NO_COLOR: "1" + # TODO-RAINCLAUDE: wait-ready emits setup_complete; the container then stays up so Antithesis can exec the test commands baked into /opt/antithesis/test/v1. + command: + - /bin/sh + - -c + - omicron-antithesis-workload wait-ready && exec sleep infinity + depends_on: + nexus: + condition: service_healthy + sled-agent-sim: + condition: service_started + networks: + underlay: + ipv6_address: fd00:1122:3344:101::7 diff --git a/antithesis/config/nexus.toml b/antithesis/config/nexus.toml new file mode 100644 index 00000000000..ab3d0de0f86 --- /dev/null +++ b/antithesis/config/nexus.toml @@ -0,0 +1,124 @@ +# TODO-RAINCLAUDE: Nexus configuration for the Antithesis deployment; derived from nexus/examples/config.toml with addresses matching antithesis/config/docker-compose.yaml. + +[console] +static_dir = "/opt/oxide/console-assets" +session_idle_timeout_minutes = 480 +session_absolute_timeout_minutes = 1440 + +[authn] +schemes_external = ["session_cookie", "access_token", "scim_token"] + +[log] +level = "info" +mode = "stderr-terminal" + +# TODO-RAINCLAUDE: no ClickHouse in phase 1; leaving the address unset keeps Nexus bootable without it. +[timeseries_db] + +[deployment] +id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" +rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc" +# TODO-RAINCLAUDE: Nexus refuses to start with no upstream resolvers (nexus/src/app/mod.rs), and the Antithesis environment has no internet, so point it at the internal DNS server: lookups of external names fail fast with NXDOMAIN instead of timing out against an unreachable public resolver. +external_dns_servers = ["fd00:1122:3344:101::3"] + +[deployment.dropshot_external] +bind_address = "[fd00:1122:3344:101::5]:12220" +default_request_body_max_bytes = 1048576 +compression = "gzip" + +[deployment.dropshot_internal] +bind_address = "[fd00:1122:3344:101::5]:12221" +default_request_body_max_bytes = 1048576 + +[deployment.dropshot_lockstep] +bind_address = "[fd00:1122:3344:101::5]:12232" +default_request_body_max_bytes = 1048576 + +[deployment.internal_dns] +type = "from_address" +address = "[fd00:1122:3344:101::3]:53" + +[deployment.database] +type = "from_url" +url = "postgresql://root@[fd00:1122:3344:101::4]:32221/omicron?sslmode=disable" + +[tunables] +max_vpc_ipv4_subnet_prefix = 26 + +[background_tasks] +dns_internal.period_secs_config = 60 +dns_internal.period_secs_servers = 60 +dns_internal.period_secs_propagation = 60 +dns_internal.max_concurrent_server_updates = 5 +dns_external.period_secs_config = 60 +dns_external.period_secs_servers = 60 +dns_external.period_secs_propagation = 60 +dns_external.max_concurrent_server_updates = 5 +metrics_producer_gc.period_secs = 60 +external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 +inventory.period_secs_load = 15 +inventory.period_secs_collect = 600 +inventory.nkeep = 5 +inventory.disable_collect = false +phantom_disks.period_secs = 30 +physical_disk_adoption.period_secs = 30 +support_bundle_collector.period_secs = 30 +decommissioned_disk_cleaner.period_secs = 60 +blueprints.period_secs_load = 10 +blueprints.period_secs_plan = 60 +blueprints.period_secs_execute = 60 +blueprints.period_secs_rendezvous = 300 +blueprints.period_secs_collect_crdb_node_ids = 180 +blueprints.period_secs_load_reconfigurator_config = 5 +switch_port_settings_manager.period_secs = 30 +region_replacement.period_secs = 30 +region_replacement_driver.period_secs = 30 +instance_watcher.period_secs = 30 +instance_updater.period_secs = 30 +instance_reincarnation.period_secs = 60 +service_firewall_propagation.period_secs = 300 +v2p_mapping_propagation.period_secs = 30 +abandoned_vmm_reaper.period_secs = 60 +saga_recovery.period_secs = 600 +lookup_region_port.period_secs = 60 +region_snapshot_replacement_start.period_secs = 30 +region_snapshot_replacement_garbage_collection.period_secs = 30 +region_snapshot_replacement_step.period_secs = 30 +region_snapshot_replacement_finish.period_secs = 30 +tuf_artifact_replication.period_secs = 300 +tuf_artifact_replication.min_sled_replication = 1 +tuf_repo_pruner.period_secs = 300 +tuf_repo_pruner.nkeep_extra_target_releases = 1 +tuf_repo_pruner.nkeep_extra_newly_uploaded = 1 +alert_dispatcher.period_secs = 60 +webhook_deliverator.period_secs = 60 +read_only_region_replacement_start.period_secs = 30 +sp_ereport_ingester.period_secs = 30 +fm.sitrep_load_period_secs = 15 +fm.config_load_period_secs = 15 +fm.analysis_period_secs = 120 +fm.sitrep_gc_period_secs = 600 +fm.sitrep_history_prune_period_secs = 600 +fm.rendezvous_period_secs = 300 +probe_distributor.period_secs = 60 +multicast_reconciler.period_secs = 60 +trust_quorum.period_secs = 60 +attached_subnet_manager.period_secs = 60 +session_cleanup.period_secs = 300 +session_cleanup.max_delete_per_activation = 10000 +audit_log_timeout_incomplete.period_secs = 600 +audit_log_timeout_incomplete.timeout_secs = 14400 +audit_log_timeout_incomplete.max_timed_out_per_activation = 1000 +audit_log_cleanup.period_secs = 600 +audit_log_cleanup.retention_days = 90 +audit_log_cleanup.max_deleted_per_activation = 10000 +populate_switch_ports.period_secs = 30 + +# TODO-RAINCLAUDE: a fixed seed keeps region allocation ordering deterministic so Antithesis controls the only randomness. +[default_region_allocation_strategy] +type = "random" +seed = 0 + +[omdb] +bin_path = "/opt/oxide/bin/omdb" diff --git a/antithesis/scratchbook/plan.md b/antithesis/scratchbook/plan.md new file mode 100644 index 00000000000..af5131bb71a --- /dev/null +++ b/antithesis/scratchbook/plan.md @@ -0,0 +1,189 @@ +# Antithesis for Nexus and CockroachDB: plan + +Date: 2026-08-27. Repo state: omicron `main` at `[9/n gen] [nexus] add typed InstanceStateGeneration (#11175)`. + +## Summary + +The goal is to run Nexus against CockroachDB inside Antithesis so that its fault injector (network partitions, congestion, node hangs, CPU throttling, clock jitter, thread pauses, and, once enabled, process kills) exercises the Nexus↔CRDB path and Nexus's own state machines: sagas, background tasks, and the instance and disk lifecycles. Properties are checked with the Antithesis Rust SDK from a workload container and from a few feature-gated hooks inside Nexus. + +Recommendation: + +1. Add an `antithesis/` harness directory to omicron laid out the way the `antithesis-skills` expect (`config/`, `scratchbook/`, `test/`, one Dockerfile), so the Antithesis skills and `snouty` work unmodified. +2. Build one multi-stage Dockerfile that produces an instrumented `omicron-antithesis` image (nexus, sled-agent-sim, dns-server, and a new workload binary, all built in a single `cargo build` with the sancov rustflags) and an `omicron-antithesis-cockroach` image from the pinned Oxide CockroachDB fork with a pre-seeded `omicron` database. +3. Phase 1 topology is five containers on an IPv6 network: `cockroach`, `internal-dns`, `nexus`, `sled-agent-sim`, `workload`. Grow to a three-node CockroachDB cluster with DNS-based discovery (phase 3), then multiple Nexus instances (phase 4). +4. Make a small set of changes to omicron proper: fix and generalize standalone `sled-agent-sim` (it is documented broken and hard-codes `::1`), add an `antithesis` cargo feature to `omicron-nexus` that turns on the SDK and instrumentation, and add three or four assertions in guaranteed code paths. +5. Write the workload in Rust as a workspace crate that reuses `oxide-client` and `nexus-lockstep-client`, exposed through thin `/opt/antithesis/test/v1/nexus/*` wrappers. + +The biggest unknowns, all resolvable in a one-week derisking phase: whether Antithesis's compose networking supports IPv6 (omicron's internal DNS is IPv6-only), whether `-Cpasses=sancov-module` works with the pinned rustc 1.97.1, and how long an instrumented release build of Nexus takes. + +## What Antithesis requires, and where it bites omicron + +Sources: [setup overview](https://antithesis.com/docs/setup/overview/), [docker compose setup](https://antithesis.com/docs/setup/docker_compose/), [test command reference](https://antithesis.com/docs/product/writing_tests/test_templates/test_composer_reference/), [Rust instrumentation](https://antithesis.com/docs/reference/sdk/rust/instrumentation/), and the [antithesis-skills](https://github.com/antithesishq/antithesis-skills) reference files. + +| Requirement | Consequence for omicron | +|---|---| +| Linux x86-64 container images, hermetic (no internet at runtime). All downloads happen in `RUN` steps. | Nexus already builds and passes its full test suite on Linux (`build-and-test (ubuntu-22.04)` job). CockroachDB, ClickHouse, dpd stub, mgd, and ddmd all have Linux artifacts via `cargo xtask download`. Nothing needed at runtime is illumos-only. | +| `docker-compose.yaml` in `antithesis/config/`; every service has `container_name` == `hostname` (no underscores), `platform: linux/amd64`, `init: true`, `image:` plus `build:`; `depends_on` with `condition: service_healthy`; no `logging:` override, no `internal: true` networks, no `pull_policy`. | Straightforward. The container names below follow this. | +| Services reach each other by hostname; Antithesis generates DNS from service names. | Omicron does not use Docker DNS. Nexus and `sled-agent-sim` need literal addresses, and internal DNS records are IPv6 (`DnsConfigBuilder::host_zone(_, Ipv6Addr)`). The compose network must be IPv6 with static `ipv6_address` per service. **Confirm IPv6 support with Antithesis in phase 0.** | +| A `setup_complete` JSON line written to `$ANTITHESIS_OUTPUT_DIR/sdk.jsonl` by any process; the first one starts testing. | Emitted by the workload container once Nexus's external API answers, login works, and the simulated sled is registered (which proves the RSS handoff completed). | +| Test commands are executables in `/opt/antithesis/test/v1/