diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 89cc0ca920..4be954eced 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -89,3 +89,12 @@ jobs: # Retry dependency preparation, then execute the test suite once. nix build --no-link .#tmachine || nix build --no-link .#tmachine nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}" + + - name: Upload tmachine diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }} + path: artifacts/tmachine-diagnostics + if-no-files-found: ignore + retention-days: 7 diff --git a/CI.md b/CI.md index a433754151..62d72f78a6 100644 --- a/CI.md +++ b/CI.md @@ -105,6 +105,12 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version. +The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so +recreating the Service requires reprovisioning the fixture. Before conformance, +every installer waits up to five minutes for a connected gateway; interactive +shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*` +artifacts. + ### Run only the policy advisor conformance tests Manually dispatch `Integration Tests` on the candidate branch with an diff --git a/tests/ansible/playbooks/conformance/cli.yaml b/tests/ansible/playbooks/conformance/cli.yaml index dd8ce2b592..11eb07e36d 100644 --- a/tests/ansible/playbooks/conformance/cli.yaml +++ b/tests/ansible/playbooks/conformance/cli.yaml @@ -5,6 +5,9 @@ - name: Run OpenShell conformance tests hosts: all gather_facts: false + vars: + # Conformance is skipped when the gateway never connects. + conformance_failed: "{{ gateway_status is failed or conformance_result.rc != 0 }}" tasks: - name: Wait for SSH ansible.builtin.wait_for_connection: @@ -55,6 +58,20 @@ register: openshell_cli changed_when: false + # Every test boot restarts the gateway, and some installers take minutes to + # recover. Wait once here; the runner's per-scenario preflight stays short. + - name: Wait for the registered gateway + ansible.builtin.command: + argv: [timeout, 10s, "{{ openshell_cli.stdout }}", status, --output, json] + register: gateway_status + changed_when: false + until: >- + gateway_status.rc == 0 and + (gateway_status.stdout | from_json).status == 'connected' + retries: 60 + delay: 5 + ignore_errors: true + - name: Run OpenShell conformance archive ansible.builtin.command: argv: @@ -77,34 +94,25 @@ register: conformance_result changed_when: false failed_when: false + when: gateway_status is succeeded # Preserve both streams for failures without adding passing-test output to # every tmachine run. - name: Show OpenShell conformance diagnostics ansible.builtin.debug: var: conformance_result - when: conformance_result.rc != 0 + when: conformance_failed - - name: Read OpenShell gateway logs - become: true - ansible.builtin.command: - argv: - - journalctl - - --no-pager - - --lines - - "500" - - _SYSTEMD_UNIT=openshell-gateway.service - - "+" - - _SYSTEMD_USER_UNIT=openshell-gateway.service - register: openshell_gateway_logs - changed_when: false - failed_when: false - when: conformance_result.rc != 0 + - name: Collect OpenShell gateway diagnostics + ansible.builtin.include_role: + name: openshell_diagnostics + when: conformance_failed - - name: Show OpenShell gateway logs - ansible.builtin.debug: - var: openshell_gateway_logs.stdout_lines - when: conformance_result.rc != 0 + - name: Require a connected gateway + ansible.builtin.assert: + that: + - gateway_status is succeeded + fail_msg: OpenShell gateway did not connect before conformance; see gateway diagnostics - name: Require OpenShell conformance success ansible.builtin.assert: diff --git a/tests/ansible/playbooks/openshell-k3s.yaml b/tests/ansible/playbooks/openshell-k3s.yaml index dd00aa4bda..591262a933 100644 --- a/tests/ansible/playbooks/openshell-k3s.yaml +++ b/tests/ansible/playbooks/openshell-k3s.yaml @@ -137,39 +137,32 @@ environment: KUBECONFIG: /etc/rancher/k3s/k3s.yaml - - name: Install gateway port-forward service - ansible.builtin.copy: - dest: /etc/systemd/system/openshell-k3s-port-forward.service - mode: "0644" - content: | - [Unit] - Description=OpenShell K3s gateway port forward - After=k3s.service - Requires=k3s.service - - [Service] - ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080 - Restart=always - RestartSec=1 - - [Install] - WantedBy=multi-user.target - - - name: Start gateway port-forward service - ansible.builtin.systemd_service: - name: openshell-k3s-port-forward.service - daemon_reload: true - enabled: true - state: started + - name: Discover the gateway ClusterIP and gRPC port + ansible.builtin.command: + argv: + - /usr/local/bin/k3s + - kubectl + - --request-timeout=10s + - --namespace + - openshell + - get + - service + - openshell + - --output=jsonpath={.spec.clusterIP}:{.spec.ports[?(@.name=="grpc")].port} + register: k3s_gateway_address + changed_when: false - - name: Wait for OpenShell gateway - ansible.builtin.wait_for: - host: 127.0.0.1 - port: 17670 - timeout: 60 + # Rejects headless Services (ClusterIP "None") and a missing grpc port. + - name: Require a gateway Service address + ansible.builtin.assert: + that: + - k3s_gateway_address.stdout is match('^[0-9.]+:[0-9]+$') + fail_msg: "Unexpected gateway Service address: {{ k3s_gateway_address.stdout }}" - name: Register OpenShell gateway for test client hosts: all gather_facts: false + vars: + openshell_client_gateway_endpoint: "http://{{ k3s_gateway_address.stdout }}" roles: - openshell_client diff --git a/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh new file mode 100644 index 0000000000..c20334bb13 --- /dev/null +++ b/tests/ansible/roles/openshell_diagnostics/files/collect-k3s.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# No kubeconfig, Secrets, environment dumps, or complete Pod specifications. +set -u +umask 077 +mkdir -p /var/lib/openshell/diagnostics +output=/var/lib/openshell/diagnostics/k3s.txt + +collect() { + printf '\n### %s\n' "$1" + shift + # Bound both stalled commands and unusually large log streams. + timeout 10s "$@" 2>&1 | tail -c 262144 + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" +} + +{ + date --utc --iso-8601=seconds + collect 'K3s service state' systemctl show k3s.service \ + -p ActiveState -p SubState -p Result -p NRestarts -p ExecMainStatus \ + -p ExecMainPID -p ActiveEnterTimestamp -p ExecMainStartTimestamp \ + -p StartLimitIntervalUSec -p StartLimitBurst -p RestartUSec + collect 'K3s boot journal' journalctl -b --no-pager --lines=500 -u k3s.service + # kube-proxy may program either iptables backend; dump whichever exist. + # shellcheck disable=SC2016 # Expanded by the inner shell. + collect 'Gateway Service routing rules' bash -c ' + for save in iptables-nft-save iptables-legacy-save; do + command -v "$save" >/dev/null || continue + echo "# $save" + "$save" 2>/dev/null | awk "/openshell\/openshell:grpc/ { print }" + done' + collect 'K3s version' /usr/local/bin/k3s --version + collect 'API readiness' /usr/local/bin/k3s kubectl --request-timeout=5s get --raw=/readyz + collect 'Nodes' /usr/local/bin/k3s kubectl --request-timeout=5s get nodes -o wide + collect 'Gateway Pod identity and state' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ + -o 'custom-columns=NAME:.metadata.name,UID:.metadata.uid,PHASE:.status.phase,CONDITIONS:.status.conditions,CONTAINERS:.status.containerStatuses' + collect 'Gateway Services' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get services \ + -o 'custom-columns=NAME:.metadata.name,TYPE:.spec.type,CLUSTERIP:.spec.clusterIP,SELECTOR:.spec.selector,PORTS:.spec.ports' + collect 'Test client gateway registrations' runuser -u tmachine -- openshell gateway list --output json + collect 'Test client gateway status' runuser -u tmachine -- openshell status --output json + collect 'Endpoint readiness' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get endpointslices \ + -o 'custom-columns=NAME:.metadata.name,PORTS:.ports,ADDRESSES:.endpoints[*].addresses,CONDITIONS:.endpoints[*].conditions' + collect 'Gateway events' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get events --sort-by=.lastTimestamp + collect 'Gateway current logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --tail=300 --timestamps + collect 'Gateway previous logs' /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs openshell-0 -c openshell-gateway --previous --tail=300 --timestamps + collect 'Memory' free -m + collect 'Disk' df -h / /var/lib/rancher/k3s +} | sed -E \ + -e 's/(Bearer )[A-Za-z0-9._~+\/-]+/\1[REDACTED]/gI' \ + -e 's/((token|password|secret|authorization)[" ]*[=:][" ]*)[^ ,"]+/\1[REDACTED]/gI' \ + > "$output" +cat "$output" diff --git a/tests/ansible/roles/openshell_diagnostics/tasks/main.yaml b/tests/ansible/roles/openshell_diagnostics/tasks/main.yaml new file mode 100644 index 0000000000..74a204ab96 --- /dev/null +++ b/tests/ansible/roles/openshell_diagnostics/tasks/main.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Collect whatever applies to the installed gateway before the VM exits. +- name: Read OpenShell gateway logs + become: true + ansible.builtin.command: + argv: + - journalctl + - --no-pager + - --lines + - "500" + - _SYSTEMD_UNIT=openshell-gateway.service + - "+" + - _SYSTEMD_USER_UNIT=openshell-gateway.service + register: openshell_gateway_logs + changed_when: false + failed_when: false + +- name: Show OpenShell gateway logs + ansible.builtin.debug: + var: openshell_gateway_logs.stdout_lines + +- name: Check for K3s + become: true + ansible.builtin.stat: + path: /usr/local/bin/k3s + register: k3s_diagnostics_binary + failed_when: false + +- name: Preserve K3s diagnostics + when: k3s_diagnostics_binary.stat.exists | default(false) + become: true + block: + - name: Collect bounded K3s diagnostics + ansible.builtin.script: collect-k3s.sh + register: k3s_diagnostics_collection + changed_when: false + failed_when: false + + - name: Show K3s diagnostics + ansible.builtin.debug: + var: k3s_diagnostics_collection.stdout_lines + + - name: Fetch K3s diagnostics + ansible.builtin.fetch: + src: /var/lib/openshell/diagnostics/k3s.txt + dest: "{{ role_path }}/../../../../artifacts/tmachine-diagnostics/{{ inventory_hostname }}/k3s.txt" + flat: true + failed_when: false