diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index d46f3fb31f..e171d2cc23 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -275,7 +275,10 @@ jobs: test-matrix: >- [ {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"} + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} ] # Run driver-specific integration tests: diff --git a/.github/workflows/e2e-docker-test.yml b/.github/workflows/e2e-docker-test.yml index f26c905d0a..b13d7d15f1 100644 --- a/.github/workflows/e2e-docker-test.yml +++ b/.github/workflows/e2e-docker-test.yml @@ -33,7 +33,7 @@ on: required: false type: string default: >- - [{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}] + [{"suite":"python","cmd":"mise run --no-deps --skip-deps e2e:python","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-python","cmd":"mise run --no-deps --skip-deps e2e:oidc-python:docker","apt_packages":"","python_proto":true,"mcp":false},{"suite":"oidc-pkce-docker","cmd":"mise run --no-deps --skip-deps e2e:oidc-pkce:docker","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"rust-docker","cmd":"mise run --no-deps --skip-deps e2e:rust","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"oci-image","cmd":"OPENSHELL_TEST_CONTAINER_ENGINE=docker e2e/with-docker-gateway.sh cargo test --locked --manifest-path tests/suites/features/Cargo.toml -p openshell-test-feature-oci-image -- --test-threads 1 --nocapture","apt_packages":"openssh-client","python_proto":false,"mcp":false},{"suite":"mcp","cmd":"mise run --no-deps --skip-deps e2e:mcp","apt_packages":"","python_proto":false,"mcp":true}] permissions: actions: read diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 0da5d1e186..f9b14bb2e7 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -145,7 +145,10 @@ jobs: test-matrix: >- [ {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"} + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} ] docker-e2e: diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 45ce4d7516..d82d571a36 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -196,7 +196,10 @@ jobs: test-matrix: >- [ {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, - {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"} + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"}, + {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"} ] docker-e2e: diff --git a/TESTING.md b/TESTING.md index a1c2f9476e..3347dac0ed 100644 --- a/TESTING.md +++ b/TESTING.md @@ -311,6 +311,30 @@ binary. `tests/artifacts.nix` keeps the follow-up exclusions explicit and uses the same filter for the generated inventory, so excluded binaries cannot appear as false passes or silently re-enter the archive. +The `oci-image` feature testsuite (`tests/suites/features/oci-image`) checks +OCI image identity and working-directory behavior shared by the Docker and +Podman drivers against installed deb packages on Ubuntu and rpm packages on +Fedora. CI runs it on Docker rootful, Podman rootful, and Podman rootless guests: + +```shell +nix run .#tmachine -- test ubuntu-docker-rootful deb oci-image +``` + +The shared suite also runs in the existing ARM64 Docker E2E workflow for +branch checks and both release workflows. That interim lane compiles the same +test crate natively and uses the architecture-matched prebuilt candidate CLI +and gateway plus the candidate multi-architecture sandbox and supervisor +images through `e2e/with-docker-gateway.sh`. The tmachine lanes still use +x86_64 artifacts; they do not replace ARM64 image coverage. + +Run it against a local gateway by naming the command that builds images into +the gateway's image store: + +```shell +OPENSHELL_TEST_CONTAINER_ENGINE=podman e2e/with-podman-gateway.sh \ + cargo test --manifest-path tests/suites/features/Cargo.toml -p openshell-test-feature-oci-image -- --test-threads 1 +``` + Run the VM-backed Rust CLI e2e suite: ```shell @@ -490,7 +514,7 @@ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test sync Run a single Docker-only test directly with cargo: ```shell -cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test custom_image +cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test docker_preflight ``` The harness (`e2e/rust/src/harness/`) provides: diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 6b1fc73723..fbab884a8d 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -58,11 +58,6 @@ name = "vm_overlay" path = "tests/vm_overlay.rs" required-features = ["e2e-vm"] -[[test]] -name = "custom_image" -path = "tests/custom_image.rs" -required-features = ["e2e-docker"] - [[test]] name = "service_bearer_passthrough" path = "tests/service_bearer_passthrough.rs" diff --git a/e2e/rust/tests/custom_image.rs b/e2e/rust/tests/custom_image.rs deleted file mode 100644 index 96fdd56612..0000000000 --- a/e2e/rust/tests/custom_image.rs +++ /dev/null @@ -1,253 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-local-container-driver")] - -//! E2E test: build custom container images and run sandboxes with them. -//! -//! Prerequisites: -//! - A running Docker- or Podman-backed openshell gateway -//! - The matching container runtime running (for image builds) -//! - The `openshell` binary (built automatically from the workspace) - -use std::{fs, io::Write}; - -use openshell_e2e::harness::container::ImageGuard; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serial_test::serial; - -const DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -# iproute2 is required for sandbox network namespace isolation. -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* - -RUN groupadd -g 1235 appstaff && \ - useradd -m -u 1234 -g appstaff app - -# The final image identity already owns the OCI working directory. Existing -# root-owned content remains root-owned. -WORKDIR /workspace/project -RUN printf root-owned > root-owned.txt && chown app:appstaff . - -# Write a marker file so we can verify this is our custom image. -# Place under /etc (Landlock baseline read-only path) so the sandbox -# can read it when filesystem restrictions are properly enforced. -RUN echo "custom-image-e2e-marker" > /etc/marker.txt - -USER app -CMD ["sleep", "infinity"] -"#; - -const NUMERIC_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* - -USER 2345:2346 -CMD ["sleep", "infinity"] -"#; - -const UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* \ - && groupadd -g 3235 appstaff \ - && useradd -m -u 3234 -g appstaff app - -WORKDIR /workspace/project -USER app -CMD ["sleep", "infinity"] -"#; - -const MARKER: &str = "custom-image-e2e-marker"; - -/// A named OCI user can write through direct and SSH children when the image -/// already grants that authority; existing content retains its ownership. -#[tokio::test] -#[serial(custom_image)] -async fn sandbox_from_custom_image() { - // Step 1: Write a temporary Dockerfile. - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let dockerfile_path = tmpdir.path().join("Dockerfile"); - { - let mut f = std::fs::File::create(&dockerfile_path).expect("create Dockerfile"); - f.write_all(DOCKERFILE_CONTENT.as_bytes()) - .expect("write Dockerfile"); - } - - // Step 2: Build the image out-of-band and create a sandbox from it. - // `--from` no longer builds local Dockerfiles itself (pre-0.1.0 - // breaking change); tests build explicitly and pass the resulting tag. - let image = ImageGuard::build("custom-dockerfile", &dockerfile_path, tmpdir.path()) - .expect("build custom image with selected container engine"); - let mut guard = SandboxGuard::create_keep_with_args( - &["--from", image.tag(), "--no-tty"], - &[ - "sh", - "-c", - "set -eu; id -u; id -g; test \"$(pwd -P)\" = /workspace/project; \ - test \"$HOME\" = /workspace/project; test \"$(cat root-owned.txt)\" = root-owned; \ - test \"$(stat -c %u:%g .)\" = 1234:1235; \ - test \"$(stat -c %u:%g root-owned.txt)\" = 0:0; \ - touch direct-oci-user-write; cat /etc/marker.txt; echo Ready; sleep infinity", - ], - "Ready", - ) - .await - .expect("sandbox create from custom image"); - - // Step 3: Verify the marker file content appears in the output. - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains(MARKER), - "expected marker '{MARKER}' in sandbox output:\n{clean_output}" - ); - assert!( - clean_output.contains("1234") && clean_output.contains("1235"), - "expected named OCI identity 1234:1235 in sandbox output:\n{clean_output}" - ); - - let ssh_output = guard - .exec(&[ - "sh", - "-c", - "set -eu; test \"$(id -u):$(id -g)\" = 1234:1235; \ - test \"$(pwd -P)\" = /workspace/project; test \"$HOME\" = /workspace/project; \ - touch ssh-oci-user-write; echo ssh-write-ok", - ]) - .await - .expect("SSH child should write to prepared workspace"); - assert!( - ssh_output.contains("ssh-write-ok"), - "expected SSH write marker:\n{ssh_output}" - ); - - let transfer_source = tmpdir.path().join("workspace-transfer.txt"); - fs::write(&transfer_source, "workspace-transfer-ok").expect("write transfer fixture"); - guard - .upload_to_workdir( - transfer_source - .to_str() - .expect("transfer fixture path is UTF-8"), - ) - .await - .expect("upload should default to the OCI workspace"); - let transfer_download = tmpdir.path().join("workspace-transfer-downloaded.txt"); - guard - .download( - "workspace-transfer.txt", - transfer_download - .to_str() - .expect("download destination path is UTF-8"), - ) - .await - .expect("download should resolve relative to the OCI workspace"); - assert_eq!( - fs::read_to_string(transfer_download).expect("read downloaded transfer fixture"), - "workspace-transfer-ok" - ); - - guard - .exec(&[ - "sh", - "-c", - "set -eu; mkdir -p merge-upload; \ - printf remote-conflict > merge-upload/conflict.txt; \ - printf remote-preserved > merge-upload/unrelated.txt", - ]) - .await - .expect("seed existing remote upload directory"); - let merge_source = tmpdir.path().join("merge-upload"); - fs::create_dir(&merge_source).expect("create local upload directory"); - fs::write(merge_source.join("conflict.txt"), "local-conflict") - .expect("write conflicting local upload file"); - fs::write(merge_source.join("added.txt"), "local-added") - .expect("write added local upload file"); - guard - .upload_to_workdir(merge_source.to_str().expect("merge upload path is UTF-8")) - .await - .expect("upload should merge into the existing remote directory"); - guard - .exec(&[ - "sh", - "-c", - "set -eu; \ - test \"$(cat merge-upload/conflict.txt)\" = local-conflict; \ - test \"$(cat merge-upload/added.txt)\" = local-added; \ - test \"$(cat merge-upload/unrelated.txt)\" = remote-preserved", - ]) - .await - .expect("upload should overwrite conflicts and preserve unrelated remote files"); - - // Explicit cleanup (also happens in Drop, but explicit is clearer in tests). - guard.cleanup().await; -} - -/// A numeric OCI user/group pair works without passwd or group entries. -/// The image intentionally has no pre-existing `/sandbox`. -#[tokio::test] -#[serial(custom_image)] -async fn sandbox_from_passwd_less_numeric_oci_user() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let dockerfile_path = tmpdir.path().join("Dockerfile"); - { - let mut f = std::fs::File::create(&dockerfile_path).expect("create Dockerfile"); - f.write_all(NUMERIC_DOCKERFILE_CONTENT.as_bytes()) - .expect("write Dockerfile"); - } - - let image = ImageGuard::build("passwd-less-numeric", &dockerfile_path, tmpdir.path()) - .expect("build numeric OCI image with selected container engine"); - let mut guard = SandboxGuard::create(&[ - "--from", - image.tag(), - "--", - "sh", - "-c", - "set -eu; id -u; id -g; test \"$(pwd -P)\" = /sandbox; \ - test \"$HOME\" = /sandbox; touch numeric-oci-user-write", - ]) - .await - .expect("sandbox create from numeric OCI Dockerfile"); - - let clean_output = strip_ansi(&guard.create_output); - assert!( - clean_output.contains("2345") && clean_output.contains("2346"), - "expected numeric OCI identity 2345:2346 in sandbox output:\n{clean_output}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(custom_image)] -async fn sandbox_rejects_image_workdir_that_would_require_new_authority() { - let tmpdir = tempfile::tempdir().expect("create tmpdir"); - let dockerfile_path = tmpdir.path().join("Dockerfile"); - fs::write(&dockerfile_path, UNWRITABLE_WORKDIR_DOCKERFILE_CONTENT).expect("write Dockerfile"); - let image = ImageGuard::build("unwritable-workdir", &dockerfile_path, tmpdir.path()) - .expect("build unwritable-workdir image with selected container engine"); - - let result = SandboxGuard::create_keep_with_args( - &["--from", image.tag(), "--no-tty"], - &["sh", "-c", "echo should-not-run"], - "should-not-run", - ) - .await; - let error = match result { - Ok(mut guard) => { - guard.cleanup().await; - panic!("root-owned workdir must not be made writable for the image user"); - } - Err(error) => error, - }; - let message = error.to_string(); - assert!( - (message.contains("WorkspaceValidationFailed") && message.contains("WorkingDir")) - || message.contains("subsystem request failed") - || message.contains("image workspace validation failed"), - "expected rejected image to fail provisioning, got: {message}" - ); -} diff --git a/e2e/rust/tests/podman_oci_identity.rs b/e2e/rust/tests/podman_oci_identity.rs index 142da82796..42e91e37c6 100644 --- a/e2e/rust/tests/podman_oci_identity.rs +++ b/e2e/rust/tests/podman_oci_identity.rs @@ -3,20 +3,21 @@ #![cfg(feature = "e2e-podman")] -//! Podman-specific E2E coverage for OCI identity inspection and immutable-image -//! launch. +//! Podman-specific E2E coverage for immutable-image launch and the isolated +//! workload/supervisor container pair. //! //! The test builds an image through the selected Podman engine, creates a -//! sandbox from its mutable tag, and verifies both the child identity and the -//! image ID recorded on the real sandbox container. This exercises the Podman -//! API inspect → protected metadata → create path rather than only its unit -//! serialization boundaries. Workspace behavior shared with Docker is covered -//! by the `oci-image` feature suite in `tests/suites/features`. +//! sandbox from its mutable tag, and inspects the real Podman containers: the +//! image ID recorded on the workload, each container's user, the supervisor's +//! capabilities, networking, and mounts. This exercises the Podman API +//! inspect → protected metadata → create path rather than only its unit +//! serialization boundaries. The sandbox identity and workspace seen by the +//! main process and `sandbox exec` are shared with Docker and covered by the +//! `oci-image` feature suite in `tests/suites/features`. use std::process::Stdio; use openshell_e2e::harness::container::{ContainerEngine, is_e2e_driver}; -use openshell_e2e::harness::output::strip_ansi; use openshell_e2e::harness::sandbox::SandboxGuard; const BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; @@ -199,42 +200,13 @@ async fn podman_uses_oci_identity_and_inspected_image_id() { std::fs::write(policy.path(), OCI_FALLBACK_POLICY).expect("write OCI fallback policy"); let policy_path = policy.path().to_str().expect("policy path is UTF-8"); let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - "--from", - &image.tag, - "--policy", - policy_path, - "--no-tty", - ], - &[ - "sh", - "-c", - "set -eu; printf 'direct-identity=%s:%s\n' \"$(id -u)\" \"$(id -g)\"; echo podman-oci-identity-ready; sleep infinity", - ], + &["--from", &image.tag, "--policy", policy_path, "--no-tty"], + &["sh", "-c", "echo podman-oci-identity-ready; sleep infinity"], READY_MARKER, ) .await .expect("create sandbox from Podman-built OCI identity image"); - let direct_output = strip_ansi(&sandbox.create_output); - assert!( - direct_output.contains("direct-identity=2345:2346"), - "expected direct child identity {OCI_UID}:{OCI_GID}:\n{direct_output}" - ); - - let ssh_output = sandbox - .exec(&[ - "sh", - "-c", - "test \"$(id -u):$(id -g)\" = 2345:2346; echo podman-ssh-identity-ok", - ]) - .await - .expect("SSH child should use Podman OCI identity"); - assert!( - ssh_output.contains("podman-ssh-identity-ok"), - "expected SSH identity marker:\n{ssh_output}" - ); - let container_id = sandbox_container_id(&image.engine, &sandbox.name).expect("find Podman sandbox container"); let launched_image_id = run_engine( diff --git a/tests/ansible/playbooks/features/oci-image.yaml b/tests/ansible/playbooks/features/oci-image.yaml new file mode 100644 index 0000000000..96cc039898 --- /dev/null +++ b/tests/ansible/playbooks/features/oci-image.yaml @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Run OCI image feature tests + hosts: all + gather_facts: false + vars: + oci_image_test_root: /var/lib/openshell-oci-image/tests + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Detect tmachine container runtime + ansible.builtin.include_role: + name: tmachine_container_runtime + + # Tests build images into the store the gateway reads. Rootful Podman's + # store belongs to root, so the unprivileged test user builds through sudo. + - name: Select the gateway's container engine command + ansible.builtin.set_fact: + oci_image_container_engine: >- + {{ 'docker' if tmachine_container_runtime_name == 'docker' + else 'podman' if tmachine_container_runtime_is_rootless + else 'sudo -n podman' }} + + - name: Create OCI image test directory + become: true + ansible.builtin.file: + path: "{{ oci_image_test_root }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Extract OCI image test bundle + become: true + ansible.builtin.unarchive: + src: "{{ oci_image_test_bundle }}" + dest: "{{ oci_image_test_root }}" + owner: tmachine + group: tmachine + + - name: Check OCI image nextest archive + ansible.builtin.stat: + path: "{{ oci_image_test_root }}/tests.tar.zst" + register: oci_image_archive + + - name: Require OCI image nextest archive + ansible.builtin.assert: + that: + - oci_image_archive.stat.isreg | default(false) + fail_msg: OCI image test bundle did not contain tests.tar.zst + + - name: Resolve installed OpenShell CLI + ansible.builtin.command: + argv: + - /bin/sh + - -c + - command -v openshell + register: openshell_cli + changed_when: false + + - name: Run OCI image archive + ansible.builtin.command: + argv: + - cargo-nextest + - nextest + - run + - --archive-file + - "{{ oci_image_test_root }}/tests.tar.zst" + - --workspace-remap + - "{{ oci_image_test_root }}" + - --no-capture + # The guest has 4 GiB; keep sandbox creates from competing for it. + - --test-threads + - "1" + - --no-fail-fast + environment: + HOME: /home/tmachine + OPENSHELL_BIN: "{{ openshell_cli.stdout }}" + OPENSHELL_TEST_CONTAINER_ENGINE: "{{ oci_image_container_engine }}" + XDG_RUNTIME_DIR: "/run/user/{{ tmachine_container_runtime_tmachine_uid.stdout }}" + register: oci_image_result + changed_when: false + failed_when: false + + - name: Show OCI image diagnostics + ansible.builtin.debug: + var: oci_image_result + when: oci_image_result.rc != 0 + + - name: Read OpenShell gateway logs + become: true + ansible.builtin.command: + argv: + - journalctl + - --unit + - openshell-gateway.service + - --no-pager + - --lines + - "500" + register: openshell_gateway_logs + changed_when: false + failed_when: false + when: oci_image_result.rc != 0 + + - name: Show OpenShell gateway logs + ansible.builtin.debug: + var: openshell_gateway_logs.stdout_lines + when: oci_image_result.rc != 0 + + - name: Require OCI image success + ansible.builtin.assert: + that: + - oci_image_result.rc == 0 + fail_msg: OCI image feature tests failed diff --git a/tests/artifacts.nix b/tests/artifacts.nix index c86d8f6d30..d63c354095 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -95,6 +95,14 @@ let target = muslToolchain.target; output = "artifacts/test-archives/${muslToolchain.target}/provider-refresh-keycloak-tests.tar"; }; + ociImageArchive = mkTestArchive { + name = "oci-image"; + workspacePath = "tests/suites/features"; + manifestPath = "tests/suites/features/Cargo.toml"; + package = "openshell-test-feature-oci-image"; + target = muslToolchain.target; + output = "artifacts/test-archives/${muslToolchain.target}/oci-image-tests.tar"; + }; # Follow-up: migrate these wrapper-coupled tests once tmachine provides their # managed-gateway controls, SPIFFE fixtures, caller driver-config setting, @@ -198,6 +206,7 @@ rec { inherit conformanceCliArchive providerRefreshKeycloakArchive + ociImageArchive podmanDriverArchive podmanE2eArchive podmanE2eCiTests @@ -246,12 +255,14 @@ rec { runtimeInputs = [ conformanceCliArchive providerRefreshKeycloakArchive + ociImageArchive podmanDriverArchive podmanE2eArchive ]; text = '' build-openshell-conformance-test-archive build-provider-refresh-keycloak-test-archive + build-oci-image-test-archive build-podman-driver-test-archive build-podman-e2e-test-archive ''; diff --git a/tests/config.nix b/tests/config.nix index 26d27b712b..1a0ce763d7 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -168,6 +168,13 @@ let provider_refresh_keycloak_test_bundle = "../artifacts/test-archives/${muslTarget}/provider-refresh-keycloak-tests.tar"; }; } + { + name = "oci-image"; + playbooks = [ "ansible/playbooks/features/oci-image.yaml" ]; + inputs = { + oci_image_test_bundle = "../artifacts/test-archives/${muslTarget}/oci-image-tests.tar"; + }; + } { name = "e2e-podman"; playbooks = [ "ansible/playbooks/drivers/podman/e2e.yaml" ]; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 8561dc97df..34d50c47fc 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -919,6 +919,15 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-test-feature-oci-image" +version = "0.0.0" +dependencies = [ + "openshell-conformance", + "tempfile", + "tokio", +] + [[package]] name = "openshell-test-feature-provider-refresh-keycloak" version = "0.0.0" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index 7acd72b5d2..9259450a16 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["provider-refresh/keycloak"] +members = ["oci-image", "provider-refresh/keycloak"] diff --git a/tests/suites/features/oci-image/Cargo.toml b/tests/suites/features/oci-image/Cargo.toml new file mode 100644 index 0000000000..1107587ca7 --- /dev/null +++ b/tests/suites/features/oci-image/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-test-feature-oci-image" +version = "0.0.0" +edition = "2024" + +[dependencies] +openshell-conformance = { path = "../../../../crates/openshell-conformance" } +tempfile = "3" +tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/suites/features/oci-image/tests/oci_image.rs b/tests/suites/features/oci-image/tests/oci_image.rs new file mode 100644 index 0000000000..175357c179 --- /dev/null +++ b/tests/suites/features/oci-image/tests/oci_image.rs @@ -0,0 +1,593 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OCI image behavior shared by the Docker and Podman compute drivers. +//! +//! Each test builds a small image with the container engine that backs the +//! target gateway, creates a sandbox from it through the candidate CLI, and +//! checks the process identity, workspace, and image content seen by both the +//! sandbox main process and `sandbox exec`. +//! +//! `OPENSHELL_BIN` names the candidate CLI. `OPENSHELL_TEST_CONTAINER_ENGINE` +//! is the command that builds images into the gateway's image store, such as +//! `docker`, `podman`, or `sudo -n podman`. + +use std::future::Future; +use std::process::Command; +use std::time::Duration; + +use openshell_conformance::OpenShellRunner; + +const BASE_IMAGE: &str = "nvcr.io/nvidia/base/ubuntu:24.04"; +const ENGINE_ENV: &str = "OPENSHELL_TEST_CONTAINER_ENGINE"; +const CREATE_TIMEOUT: Duration = Duration::from_mins(10); +const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); +/// A complete sandbox policy without a `process` section, so the sandbox +/// identity falls back to the image `USER`. +const IMAGE_IDENTITY_POLICY: &str = "version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /lib64, /proc, /dev/urandom, /etc] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort + +network_policies: {} +"; + +/// A named image user that owns its custom `WORKDIR`. Existing image content +/// keeps its ownership. +#[tokio::test] +async fn custom_workdir_with_named_user() { + run( + "oci-image/custom-workdir-named-user", + async |runner, images| { + let image = TestImage::build( + images, + "named-workdir", + &format!( + "FROM {BASE_IMAGE} +RUN groupadd -g 1235 appstaff && useradd -m -u 1234 -g appstaff app +WORKDIR /workspace/project +RUN printf root-owned > root-owned.txt && chown app:appstaff . && \ + printf custom-image-e2e-marker > /etc/oci-image-marker +USER app +" + ), + )?; + let checks = format!( + "{} test \"$(stat -c %u:%g .)\" = 1234:1235; \ + test \"$(cat /etc/oci-image-marker)\" = custom-image-e2e-marker; \ + test \"$(stat -c %u:%g /etc/oci-image-marker)\" = 0:0;", + workspace_checks("1234:1235", "/workspace/project", true) + ); + let sandbox = create_sandbox(runner, "named", "nu", image, None, &checks).await?; + file_transfer_uses_workspace(runner, &sandbox).await + }, + ) + .await; +} + +/// A numeric image user without passwd entries can reach a custom `WORKDIR` +/// whose parent directories are private to that user. The sandbox policy omits +/// `process`, so the image `USER` is the only source of the sandbox identity. +#[tokio::test] +async fn custom_workdir_with_numeric_user_and_private_parents() { + run( + "oci-image/custom-workdir-numeric-user", + async |runner, images| { + let image = TestImage::build( + images, + "numeric-workdir", + &format!( + "FROM {BASE_IMAGE} +RUN mkdir -p /home/app/project && \\ + chown 2345:2346 /home/app /home/app/project && \\ + chmod 0700 /home/app /home/app/project +WORKDIR /home/app/project +RUN printf root-owned > root-owned.txt +USER 2345:2346 +" + ), + )?; + let policy_file = tempfile::NamedTempFile::new() + .map_err(|error| format!("create policy file: {error}"))?; + std::fs::write(policy_file.path(), IMAGE_IDENTITY_POLICY) + .map_err(|error| format!("write policy file: {error}"))?; + let policy = policy_file + .path() + .to_str() + .ok_or("policy path is not UTF-8")?; + let checks = workspace_checks("2345:2346", "/home/app/project", true); + create_sandbox(runner, "numeric", "pu", image, Some(policy), &checks) + .await + .map(drop) + }, + ) + .await; +} + +/// An image without a `WORKDIR` uses the managed `/sandbox` workspace, owned +/// by the image user, even when the image does not contain `/sandbox`. +#[tokio::test] +async fn default_workdir_uses_managed_workspace() { + run("oci-image/default-workdir", async |runner, images| { + let image = TestImage::build( + images, + "default-workdir", + &format!("FROM {BASE_IMAGE}\nUSER 2345:2346\n"), + )?; + let checks = workspace_checks("2345:2346", "/sandbox", false); + create_sandbox(runner, "default", "dw", image, None, &checks) + .await + .map(drop) + }) + .await; +} + +/// OpenShell rejects a custom `WORKDIR` that the image user cannot write +/// instead of granting the user new access to it. Drivers may surface the +/// rejection through different startup diagnostics rather than one condition +/// reason. +#[tokio::test] +async fn unwritable_custom_workdir_is_rejected() { + run("oci-image/unwritable-workdir", async |runner, images| { + // First prove that this user, workspace and command can run. Only the + // directory owner differs between the control and rejected image. + for (suffix, owner, writable) in [("wc", "3234:3235", true), ("uw", "0:0", false)] { + let image = TestImage::build( + images, + suffix, + &format!( + "FROM {BASE_IMAGE} +RUN groupadd -g 3235 appstaff && useradd -m -u 3234 -g appstaff app +WORKDIR /workspace/project +RUN chown {owner} . +USER app +" + ), + )?; + let name = format!("oi-{}-{suffix}", runner.id()); + runner.track_sandbox(&name); + let create = runner + .step(format!("{suffix}/create")) + .description(if writable { + "writable control runs the same command successfully" + } else { + "sandbox creation fails before the command runs" + }) + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + &name, + "--from", + &image.tag, + "--no-tty", + "--", + "sh", + "-c", + "echo workspace-access-marker", + ]) + .await + .map_err(|error| error.to_string())?; + if writable { + create.require_success()?; + if !create.stdout().contains("workspace-access-marker") { + return Err( + create.failure_diagnostic("writable control executes the command marker") + ); + } + continue; + } + if create.success() + || create.stdout().contains("workspace-access-marker") + || create.stderr().contains("workspace-access-marker") + { + return Err( + create.failure_diagnostic("sandbox creation fails before the command runs") + ); + } + let diagnostic = format!("{}\n{}", create.stdout(), create.stderr()); + if !has_workspace_rejection_diagnostic(&diagnostic) { + return Err(create.failure_diagnostic( + "sandbox creation reports a workspace, permission, or workload startup rejection", + )); + } + } + Ok(()) + }) + .await; +} + +fn has_workspace_rejection_diagnostic(diagnostic: &str) -> bool { + let diagnostic = diagnostic.to_ascii_lowercase(); + // A successful control does not excuse a later connectivity or image-pull + // failure. Those are not evidence of workspace rejection, even if their + // messages happen to mention the workspace. + if [ + "connection refused", + "connection reset", + "imagepull", + "image pull", + "failed to pull", + "pull access denied", + "manifest unknown", + "no such image", + ] + .iter() + .any(|message| diagnostic.contains(message)) + { + return false; + } + // The CLI may wrap the human message across lines with diagnostic gutters. + // Match the existing startup reason and exit detail independently. + if diagnostic.contains("containerexited") && diagnostic.contains("exited with code") { + return true; + } + [ + "workspace", + "workingdir", + "permission denied", + // Some drivers expose the rejected workload launch through SSH rather + // than propagating the runtime's workspace validation text. + "subsystem request failed", + ] + .iter() + .any(|message| diagnostic.contains(message)) +} + +#[test] +fn workspace_rejection_diagnostics_do_not_require_one_condition_reason() { + for diagnostic in [ + "image workspace validation failed", + "WorkingDir /workspace/project is not writable", + "Permission denied (os error 13)", + "ContainerExited: Container exited with code 1", + "Error: × sandbox entered error phase while provisioning: ContainerExited: Container\n │ exited with code 1", + "subsystem request failed", + ] { + assert!(has_workspace_rejection_diagnostic(diagnostic)); + } + for diagnostic in [ + "", + "gateway connection refused", + "image pull failed", + "image pull failed for workspace fixture", + "gateway connection refused while creating workspace", + "ImagePullFailed: failed to pull image for WorkingDir test", + "ContainerExited", + ] { + assert!(!has_workspace_rejection_diagnostic(diagnostic)); + } +} + +async fn run( + scenario: &str, + test: impl AsyncFnOnce(&mut OpenShellRunner, &mut Vec) -> Result<(), String>, +) { + let mut runner = + OpenShellRunner::from_env(scenario).expect("candidate openshell CLI is available"); + let mut images = Vec::new(); + let result = async { + runner.check_gateway_status().await?; + test(&mut runner, &mut images).await + } + .await; + if let Err(error) = + finish_with_image_cleanup(runner.finish(result), &images, TestImage::remove).await + { + panic!("{scenario} failed:\n{error}"); + } +} + +/// Finish OpenShell sandbox cleanup before removing image tags. Non-forced +/// image removal must report remaining users rather than deleting workloads +/// behind the gateway's back. Keep functional/sandbox failures primary. +async fn finish_with_image_cleanup( + finish: impl Future>, + images: &[T], + mut remove: impl FnMut(&T) -> Result<(), String>, +) -> Result<(), String> { + let result = finish.await; + let errors: Vec<_> = images + .iter() + .filter_map(|image| remove(image).err()) + .collect(); + if errors.is_empty() { + return result; + } + let cleanup = format!("image cleanup failed:\n{}", errors.join("\n")); + match result { + Ok(()) => Err(cleanup), + Err(primary) => Err(format!("{primary}\n{cleanup}")), + } +} + +#[tokio::test] +async fn sandbox_cleanup_precedes_image_removal_on_success_and_failure() { + use std::cell::RefCell; + + for fail_scenario in [false, true] { + for fail_cleanup in [false, true] { + let events = RefCell::new(Vec::new()); + let finish = async { + events.borrow_mut().push("sandbox delete"); + if fail_scenario { + Err("functional failure".to_string()) + } else { + Ok(()) + } + }; + let result = finish_with_image_cleanup(finish, &["first", "second"], |image| { + events.borrow_mut().push(image); + if fail_cleanup { + Err(format!("cannot remove {image}")) + } else { + Ok(()) + } + }) + .await; + assert_eq!(*events.borrow(), ["sandbox delete", "first", "second"]); + assert_eq!(result.is_err(), fail_scenario || fail_cleanup); + if fail_scenario { + assert!( + result + .as_ref() + .unwrap_err() + .starts_with("functional failure") + ); + } + if fail_cleanup { + let error = result.unwrap_err(); + assert!(error.contains("cannot remove first")); + assert!(error.contains("cannot remove second")); + } + } + } +} + +/// Shell checks for the identity, working directory, and `HOME` of a sandbox +/// child. With `image_file`, also check that root-owned image content is +/// present and unchanged in the workspace. +fn workspace_checks(identity: &str, workspace: &str, image_file: bool) -> String { + let mut checks = format!( + "test \"$(id -u):$(id -g)\" = {identity}; \ + test \"$(pwd -P)\" = {workspace}; \ + test \"$HOME\" = {workspace};" + ); + if image_file { + checks.push_str( + " test \"$(cat root-owned.txt)\" = root-owned; \ + test \"$(stat -c %u:%g root-owned.txt)\" = 0:0;", + ); + } + checks +} + +/// Create a detached sandbox whose main process runs `checks` and writes to +/// the workspace, then run the same checks and a write through `sandbox exec`. +async fn create_sandbox( + runner: &mut OpenShellRunner, + suffix: &str, + short: &str, + image: &TestImage, + policy: Option<&str>, + checks: &str, +) -> Result { + // Sandbox names are limited to 19 characters on some drivers. + let name = format!("oi-{}-{short}", runner.id()); + runner.track_sandbox(&name); + let main = format!( + "(set -eu; {checks} touch main-write) >/tmp/oci-main.log 2>&1; \ + echo $? >/tmp/oci-main.status; exec sleep infinity" + ); + let mut args = vec!["sandbox", "create", "--name", &name, "--from", &image.tag]; + if let Some(policy) = policy { + args.extend(["--policy", policy]); + } + args.extend(["--detach", "--", "sh", "-c", &main]); + runner + .step(format!("{suffix}/create")) + .description("sandbox starts from the test image") + .with_timeout(CREATE_TIMEOUT) + .run(&args) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let exec = format!( + "set -eu; i=0; \ + while [ ! -f /tmp/oci-main.status ]; do \ + i=$((i + 1)); [ \"$i\" -le 60 ] || {{ echo main process checks did not finish >&2; exit 1; }}; \ + sleep 1; \ + done; \ + if [ \"$(cat /tmp/oci-main.status)\" != 0 ]; then \ + echo main process checks failed: >&2; cat /tmp/oci-main.log >&2; exit 1; \ + fi; \ + test -f main-write; {checks} touch exec-write" + ); + runner + .step(format!("{suffix}/exec")) + .description("main process and exec children see the image workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", "exec", "--name", &name, "--no-tty", "--", "sh", "-c", &exec, + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + Ok(name) +} + +/// Upload and download default to paths relative to the image workspace. +async fn file_transfer_uses_workspace( + runner: &OpenShellRunner, + sandbox: &str, +) -> Result<(), String> { + let local = tempfile::tempdir().map_err(|error| format!("create temp dir: {error}"))?; + let upload = local.path().join("oci-transfer.txt"); + std::fs::write(&upload, "oci-transfer-ok").map_err(|error| format!("write upload: {error}"))?; + let upload = upload.to_str().ok_or("upload path is not UTF-8")?; + runner + .step("named/upload") + .description("upload without a destination writes to the workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "upload", sandbox, upload, "--no-git-ignore"]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + runner + .step("named/uploaded") + .description("uploaded file is in the workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + sandbox, + "--no-tty", + "--", + "sh", + "-c", + "test \"$(cat oci-transfer.txt)\" = oci-transfer-ok", + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + + let download = local.path().join("downloaded.txt"); + let download_path = download.to_str().ok_or("download path is not UTF-8")?; + runner + .step("named/download") + .description("download resolves relative paths in the workspace") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "download", + sandbox, + "oci-transfer.txt", + download_path, + ]) + .await + .map_err(|error| error.to_string())? + .require_success()?; + let downloaded = + std::fs::read_to_string(&download).map_err(|error| format!("read download: {error}"))?; + if downloaded != "oci-transfer-ok" { + return Err(format!( + "downloaded file has unexpected content: {downloaded:?}" + )); + } + + // A directory upload merges into an existing workspace directory. + let merge = local.path().join("merge-upload"); + std::fs::create_dir(&merge).map_err(|error| format!("create merge dir: {error}"))?; + std::fs::write(merge.join("conflict.txt"), "local-conflict") + .and_then(|()| std::fs::write(merge.join("added.txt"), "local-added")) + .map_err(|error| format!("write merge files: {error}"))?; + let merge = merge.to_str().ok_or("merge path is not UTF-8")?; + let seed = "mkdir merge-upload && printf remote-conflict > merge-upload/conflict.txt \ + && printf remote-preserved > merge-upload/unrelated.txt"; + let verify = "test \"$(cat merge-upload/conflict.txt)\" = local-conflict \ + && test \"$(cat merge-upload/added.txt)\" = local-added \ + && test \"$(cat merge-upload/unrelated.txt)\" = remote-preserved"; + for (step, description, args) in [ + ( + "named/merge-seed", + "seed an existing workspace directory", + [ + "sandbox", "exec", "--name", sandbox, "--no-tty", "--", "sh", "-c", seed, + ] + .as_slice(), + ), + ( + "named/merge-upload", + "directory upload merges into the existing directory", + ["sandbox", "upload", sandbox, merge, "--no-git-ignore"].as_slice(), + ), + ( + "named/merged", + "upload overwrites conflicts and keeps unrelated files", + [ + "sandbox", "exec", "--name", sandbox, "--no-tty", "--", "sh", "-c", verify, + ] + .as_slice(), + ), + ] { + runner + .step(step) + .description(description) + .with_timeout(COMMAND_TIMEOUT) + .run(args) + .await + .map_err(|error| error.to_string())? + .require_success()?; + } + Ok(()) +} + +/// An image owned by `run`, kept alive until sandbox cleanup completes. +struct TestImage { + engine: Vec, + tag: String, +} + +impl TestImage { + fn build<'a>( + images: &'a mut Vec, + name: &str, + containerfile: &str, + ) -> Result<&'a Self, String> { + let engine: Vec = std::env::var(ENGINE_ENV) + .map_err(|_| format!("{ENGINE_ENV} must name the gateway's container engine"))? + .split_whitespace() + .map(str::to_string) + .collect(); + if engine.is_empty() { + return Err(format!("{ENGINE_ENV} is empty")); + } + let context = tempfile::tempdir().map_err(|error| format!("create context: {error}"))?; + let file = context.path().join("Containerfile"); + std::fs::write(&file, containerfile) + .map_err(|error| format!("write Containerfile: {error}"))?; + images.push(Self { + engine, + tag: format!("localhost/openshell-test-oci-{name}:{}", std::process::id()), + }); + let image = images.last().expect("registered image"); + image.engine_command(&[ + "build", + "--file", + file.to_str().ok_or("Containerfile path is not UTF-8")?, + "--tag", + &image.tag, + context.path().to_str().ok_or("context path is not UTF-8")?, + ])?; + Ok(image) + } + + fn remove(&self) -> Result<(), String> { + self.engine_command(&["image", "rm", &self.tag]) + } + + fn engine_command(&self, args: &[&str]) -> Result<(), String> { + let command = format!("{} {}", self.engine.join(" "), args.join(" ")); + let output = Command::new(&self.engine[0]) + .args(&self.engine[1..]) + .args(args) + .output() + .map_err(|error| format!("failed to run {command}: {error}"))?; + if !output.status.success() { + return Err(format!( + "{command} failed ({}):\n{}{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(()) + } +}