diff --git a/.github/workflows/gpu-compiler-experiment.yml b/.github/workflows/gpu-compiler-experiment.yml new file mode 100644 index 00000000000..eefae4ed00e --- /dev/null +++ b/.github/workflows/gpu-compiler-experiment.yml @@ -0,0 +1,401 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Temporary draft PR experiment. No package or image is published. +name: Experiment / GPU Rust compiler + +on: + # NVIDIA self-hosted runners accept trusted branch pushes, not PR events. + push: + branches: [experiment/gpu-rust-compiler] + paths: + - .github/workflows/gpu-compiler-experiment.yml + - experiments/gpu-rust-compiler/** + # A new workflow starts from the branch push. Manual dispatch requires the + # workflow on the repository's default branch; this PR will not be merged. + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: gpu-compiler-experiment-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CANDIDATE_SHA: ${{ github.sha }} + CARGO_INCREMENTAL: "0" + +jobs: + native-cpu: + name: Experiment / Native CPU frontend + if: >- + github.repository == 'NVIDIA/NemoClaw' && + github.ref == 'refs/heads/experiment/gpu-rust-compiler' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-24.04 + timeout-minutes: 40 + permissions: + contents: read + pull-requests: read + outputs: + frontend-artifact: ${{ steps.identity.outputs.artifact }} + steps: + - name: Require the open draft experiment PR at this exact head + working-directory: . + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api repos/NVIDIA/NemoClaw/pulls/12951 | jq --exit-status \ + --arg sha "$CANDIDATE_SHA" ' + .number == 12951 and .state == "open" and .draft == true + and .base.repo.full_name == "NVIDIA/NemoClaw" and .base.ref == "v1" + and .head.repo.full_name == "NVIDIA/NemoClaw" + and .head.ref == "experiment/gpu-rust-compiler" and .head.sha == $sha + ' > /dev/null + + - name: Check out the exact experiment head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.CANDIDATE_SHA }} + persist-credentials: false + + - name: Verify source and record build tools + working-directory: experiments/gpu-rust-compiler + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + mkdir -p results .build/ci-native + printf '%s\n' "$CANDIDATE_SHA" > .build/ci-native/source-sha.txt + rustup toolchain install 1.98.1 --profile minimal + rustc +1.98.1 --version --verbose > results/rustc.txt + clang --version > results/clang-native.txt + + - name: Build and test the independent frontend + working-directory: experiments/gpu-rust-compiler + run: | + set -euo pipefail + cargo +1.98.1 test --locked --manifest-path compiler/Cargo.toml --target-dir .build/compiler + cargo +1.98.1 build --locked --release --bins --manifest-path compiler/Cargo.toml --target-dir .build/compiler + .build/compiler/release/gpu-rust-compiler fixtures/compiler_demo.rs \ + --backend cpu --verify --output .build/compiler-demo-cpu \ + --report results/native-cpu-demo.json + .build/compiler-demo-cpu > results/native-cpu-demo.stdout + test "$(cat results/native-cpu-demo.stdout)" = 1460 + cp .build/compiler/release/gpu-rust-compiler .build/ci-native/gpu-rust-compiler + cp .build/compiler/release/gpu-cpu-liveness .build/ci-native/gpu-cpu-liveness + cp .build/compiler/release/gpu-native-benchmark .build/ci-native/gpu-native-benchmark + cp .build/compiler/release/gpu-cargo .build/ci-native/gpu-cargo + cp .build/compiler/release/gpu-bitcode-linker .build/ci-native/gpu-bitcode-linker + cp .build/compiler/release/gpu-codegen .build/ci-native/gpu-codegen + cp .build/compiler/release/gpu-emit-module .build/ci-native/gpu-emit-module + cp .build/compiler/release/gpu-verify-pipeline .build/ci-native/gpu-verify-pipeline + cp .build/compiler/release/gpu-corpus-census .build/ci-native/gpu-corpus-census + cp .build/compiler/release/gpu-corpus-replay .build/ci-native/gpu-corpus-replay + cargo +1.98.1 test --locked --manifest-path fast-backend/Cargo.toml --target-dir .build/fast-controller + cargo +1.98.1 build --locked --release --bins --manifest-path fast-backend/Cargo.toml --target-dir .build/fast-controller + cp .build/fast-controller/release/nemoclaw-fast-backend .build/ci-native/nemoclaw-fast-backend + cp .build/fast-controller/release/rust-llvm-llc .build/ci-native/rust-llvm-llc + RUSTUP_TOOLCHAIN=1.98.1 .build/compiler/release/gpu-reference-programs .build/ci-native/references + (cd .build/ci-native && sha256sum gpu-rust-compiler gpu-cpu-liveness \ + gpu-native-benchmark gpu-cargo gpu-bitcode-linker gpu-codegen gpu-emit-module \ + gpu-verify-pipeline gpu-corpus-census gpu-corpus-replay \ + nemoclaw-fast-backend rust-llvm-llc source-sha.txt references/* > binary.sha256) + + - name: Identify the exact-head frontend artifact + id: identity + working-directory: experiments/gpu-rust-compiler + run: | + printf 'artifact=gpu-compiler-frontend-%s-%s-%s\n' \ + "$CANDIDATE_SHA" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" + + - name: Upload the frontend for this run + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.identity.outputs.artifact }} + path: experiments/gpu-rust-compiler/.build/ci-native/ + include-hidden-files: true + if-no-files-found: error + retention-days: 14 + + - name: Retain native CPU evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-compiler-native-cpu-${{ env.CANDIDATE_SHA }}-${{ github.run_attempt }} + path: experiments/gpu-rust-compiler/results/ + if-no-files-found: warn + retention-days: 14 + + cpu-benchmark: + name: Experiment / Complete NemoClaw CPU benchmark + needs: native-cpu + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - name: Check out the exact experiment head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.CANDIDATE_SHA }} + persist-credentials: false + + - name: Download this run's exact-head frontend + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.native-cpu.outputs.frontend-artifact }} + path: experiments/gpu-rust-compiler/.build/compiler/release/ + + - name: Verify source and CPU frontend provenance + working-directory: experiments/gpu-rust-compiler + env: + WORKFLOW_SHA: ${{ github.workflow_sha }} + FRONTEND_ARTIFACT: ${{ needs.native-cpu.outputs.frontend-artifact }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + test "$(cat .build/compiler/release/source-sha.txt)" = "$CANDIDATE_SHA" + (cd .build/compiler/release && sha256sum --check binary.sha256) + chmod +x .build/compiler/release/gpu-cargo + mkdir -p results + rustup toolchain install 1.98.1 --profile minimal + rustc +1.98.1 --version --verbose > results/rustc-full-nemoclaw-cpu.txt + jq --null-input \ + --arg candidate "$CANDIDATE_SHA" --arg workflow "$WORKFLOW_SHA" \ + --arg run "$GITHUB_RUN_ID" --arg attempt "$GITHUB_RUN_ATTEMPT" \ + --arg frontend "$FRONTEND_ARTIFACT" \ + '{candidate_sha:$candidate,workflow_sha:$workflow,run_id:$run,run_attempt:$attempt,frontend_artifact:$frontend,scope:"Complete NemoClaw release and fast CPU policy comparison"}' \ + > results/full-nemoclaw-cpu-provenance.json + + - name: Set up the compatible NemoClaw build contract + uses: ./.github/actions/setup-rust + with: + platform: linux_amd64 + cache-prefix: gpu-experiment-full-rust-v1 + save-cache: "false" + + - name: Build complete NemoClaw in release and fast CPU policies + working-directory: . + run: | + set -euo pipefail + exp="$PWD/experiments/gpu-rust-compiler" + "$exp/.build/compiler/release/gpu-cargo" build --manifest-path "$PWD/Cargo.toml" \ + --package nemoclaw-cli --bin nemoclaw --quality release --placement cpu \ + --target-root "$exp/.build/full-nemoclaw" --report "$exp/results/full-nemoclaw-release.json" + reference=$(jq -r .artifact.path "$exp/results/full-nemoclaw-release.json") + "$exp/.build/compiler/release/gpu-cargo" build --manifest-path "$PWD/Cargo.toml" \ + --package nemoclaw-cli --bin nemoclaw --quality fast --placement cpu \ + --target-root "$exp/.build/full-nemoclaw" --reference-artifact "$reference" \ + --max-size-ratio 1.5 --size-basis strip-debug --report "$exp/results/full-nemoclaw-fast.json" + candidate=$(jq -r .artifact.path "$exp/results/full-nemoclaw-fast.json") + "$candidate" --version > "$exp/results/full-nemoclaw-fast-version.txt" + + - name: Retain complete NemoClaw CPU evidence, including failures + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-compiler-full-nemoclaw-cpu-${{ env.CANDIDATE_SHA }}-${{ github.run_attempt }} + path: experiments/gpu-rust-compiler/results/ + if-no-files-found: warn + retention-days: 14 + + cuda: + name: Experiment / CUDA on RTX PRO 6000 + needs: native-cpu + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 60 + env: + CONTAINER_IMAGE: gpu-rust-compiler-experiment:${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: Check out the exact experiment head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.CANDIDATE_SHA }} + persist-credentials: false + + - name: Download this run's exact-head frontend + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.native-cpu.outputs.frontend-artifact }} + path: experiments/gpu-rust-compiler/.build/compiler/release/ + + - name: Verify source and frontend provenance + working-directory: experiments/gpu-rust-compiler + env: + WORKFLOW_SHA: ${{ github.workflow_sha }} + EVENT_NAME: ${{ github.event_name }} + RUNNER_NAME: ${{ runner.name }} + RUNNER_ARCH: ${{ runner.arch }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + test "$(cat .build/compiler/release/source-sha.txt)" = "$CANDIDATE_SHA" + (cd .build/compiler/release && sha256sum --check binary.sha256) + chmod +x .build/compiler/release/gpu-rust-compiler \ + .build/compiler/release/gpu-cpu-liveness .build/compiler/release/gpu-native-benchmark \ + .build/compiler/release/gpu-cargo .build/compiler/release/gpu-bitcode-linker \ + .build/compiler/release/gpu-codegen .build/compiler/release/gpu-emit-module \ + .build/compiler/release/gpu-verify-pipeline .build/compiler/release/nemoclaw-fast-backend \ + .build/compiler/release/rust-llvm-llc .build/compiler/release/gpu-corpus-census \ + .build/compiler/release/gpu-corpus-replay + chmod +x .build/compiler/release/references/*-reference + mkdir -p results .build/verification/cuda + jq --null-input \ + --arg candidate "$CANDIDATE_SHA" --arg workflow "$WORKFLOW_SHA" \ + --arg event "$EVENT_NAME" --arg run "$GITHUB_RUN_ID" \ + --arg attempt "$GITHUB_RUN_ATTEMPT" --arg runner "$RUNNER_NAME" \ + --arg architecture "$RUNNER_ARCH" \ + '{candidate_sha:$candidate,workflow_sha:$workflow,event:$event,run_id:$run,run_attempt:$attempt,runner:$runner,runner_arch:$architecture,scope:"Full Rust bitcode backend, GPU leaf object emission, and scalar liveness experiments; no full NemoClaw GPU speedup claim."}' \ + > results/ci-provenance.json + + - name: Record the GPU and container engine + working-directory: experiments/gpu-rust-compiler + run: | + set -euo pipefail + nvidia-smi > results/nvidia-smi-host.txt 2>&1 + docker version > results/docker-version.txt 2>&1 + + - name: Build the local CUDA validation environment + working-directory: experiments/gpu-rust-compiler + run: | + set -euo pipefail + docker build --pull --file cuda/validation.Dockerfile \ + --tag "$CONTAINER_IMAGE" cuda 2>&1 | tee results/container-build.log + docker image inspect "$CONTAINER_IMAGE" > results/container-image.json + + - name: Fetch locked Rust inputs before the offline corpus audit + working-directory: experiments/gpu-rust-compiler + run: | + set -euo pipefail + mkdir -p .build/corpus-cargo + docker run --rm --read-only \ + --cap-drop ALL --security-opt no-new-privileges \ + --user "$(id -u):$(id -g)" \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=1g,mode=1777 \ + --mount "type=bind,source=$PWD/../..,target=/workspace,readonly" \ + --mount "type=bind,source=$PWD/.build,target=/lab/.build" \ + --env CARGO_HOME=/lab/.build/corpus-cargo \ + "$CONTAINER_IMAGE" bash --noprofile --norc -euo pipefail -c ' + cargo +1.98.1 fetch --locked --manifest-path /workspace/Cargo.toml + ' 2>&1 | tee results/corpus-fetch.log + + - name: Verify CUDA and compile the supported subset + working-directory: experiments/gpu-rust-compiler + run: | + set -euo pipefail + docker run --rm --gpus all --network none --read-only \ + --cap-drop ALL --security-opt no-new-privileges \ + --user "$(id -u):$(id -g)" \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=1g,mode=1777 \ + --mount "type=bind,source=$PWD,target=/lab,readonly" \ + --mount "type=bind,source=$PWD/../..,target=/workspace,readonly" \ + --mount "type=bind,source=$PWD/.build,target=/lab/.build" \ + --mount "type=bind,source=$PWD/results,target=/lab/results" \ + --workdir /lab --env CANDIDATE_SHA \ + --env CARGO_HOME=/lab/.build/corpus-cargo --env CARGO_INCREMENTAL=0 \ + --env PYTHONUNBUFFERED=1 --env PYTHONDONTWRITEBYTECODE=1 \ + "$CONTAINER_IMAGE" bash --noprofile --norc -euo pipefail -c ' + nvcc --version > results/nvcc-container.txt + clang --version > results/clang-container.txt + dpkg-query -W > results/container-packages.txt + cmake -S fast-backend -B .build/tpde-native -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_DIR=/opt/llvm/lib/cmake/llvm \ + -DFETCHCONTENT_SOURCE_DIR_TPDE=/opt/tpde-source \ + -DFETCHCONTENT_FULLY_DISCONNECTED=ON \ + -DCMAKE_CXX_COMPILER=/opt/llvm/bin/clang++ -DCMAKE_C_COMPILER=/opt/llvm/bin/clang + cmake --build .build/tpde-native --target tpde-rust-bridge --parallel 4 + nvcc -O3 -std=c++17 -shared -Xcompiler -fPIC native/codegen_cuda.cu \ + -o .build/verification/cuda/libgpuemit_cuda.so + GPU_LINK_CAPTURE_DIR=/lab/.build/nemoclaw-corpus \ + .build/compiler/release/gpu-cargo build --manifest-path /workspace/Cargo.toml \ + --package nemoclaw-cli --bin nemoclaw --quality fast --placement cpu --offline \ + --object-backend llvm-bitcode --bitcode-wrapper .build/compiler/release/gpu-bitcode-linker \ + --tpde-controller .build/compiler/release/nemoclaw-fast-backend \ + --tpde-bridge .build/tpde-native/tpde-rust-bridge \ + --llvm-llc .build/compiler/release/rust-llvm-llc \ + --target-root /lab/.build/nemoclaw-corpus-target \ + --report /lab/results/nemoclaw-corpus-build.json + candidate=$(jq -r .artifact.path results/nemoclaw-corpus-build.json) + "$candidate" --version > results/nemoclaw-corpus-version.txt + .build/compiler/release/gpu-corpus-census \ + --corpus-root .build/nemoclaw-corpus --build-report results/nemoclaw-corpus-build.json \ + --bridge .build/tpde-native/tpde-rust-bridge \ + --packet-dir .build/nemoclaw-corpus-packets --report results/nemoclaw-corpus-census.json + eligible=$(jq -r .eligible_units results/nemoclaw-corpus-census.json) + if [ "$eligible" -gt 0 ]; then + .build/compiler/release/gpu-corpus-replay --manifest results/nemoclaw-corpus-census.json \ + --backend cuda --library .build/verification/cuda/libgpuemit_cuda.so \ + --devices 0 --cpu-workers 4 --repeats 5 --report results/nemoclaw-corpus-replay.json + fi + .build/compiler/release/gpu-codegen --backend cuda --functions 128 \ + --batch-functions 32 --repeats 5 --cpu-workers 4 --devices 0 \ + --library .build/verification/cuda/libgpuemit_cuda.so \ + --out-dir .build/verification/codegen-cuda --execute + .build/compiler/release/gpu-codegen --backend cpu --functions 128 \ + --batch-functions 32 --repeats 5 --cpu-workers 4 \ + --out-dir .build/verification/codegen-cpu --execute + .build/compiler/release/gpu-verify-pipeline --backend cuda \ + --fixture-manifest fixtures/full-rust-gpu/Cargo.toml \ + --gpu-cargo .build/compiler/release/gpu-cargo \ + --wrapper .build/compiler/release/gpu-bitcode-linker \ + --controller .build/compiler/release/nemoclaw-fast-backend \ + --bridge .build/tpde-native/tpde-rust-bridge --llc .build/compiler/release/rust-llvm-llc \ + --gpu-emitter .build/compiler/release/gpu-emit-module \ + --gpu-library .build/verification/cuda/libgpuemit_cuda.so \ + --target-root .build/verification/full-rust-cuda --report results/full-rust-cuda-verified.json + python3 -m unittest discover -s tests -p test_liveness_reference.py + python3 -m unittest discover -s tests -p test_prepare_workload.py + python3 -m unittest discover -s tests -p test_benchmark_cuda.py + python3 -m unittest discover -s tests -p test_native_cuda_evidence.py + python3 scripts/verify_cuda.py --repeats 15 --report results/cuda-verification.json + python3 scripts/compile.py fixtures/compiler_demo.rs --backend cpu \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --output .build/compiler-demo-cpu-bridge --report results/cpu-bridge-demo.json + .build/compiler-demo-cpu-bridge > results/cpu-bridge-demo.stdout + test "$(cat results/cpu-bridge-demo.stdout)" = 1460 + python3 scripts/compile.py fixtures/compiler_demo.rs --backend cuda \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --cuda-binary .build/cuda/cuda-liveness \ + --output .build/compiler-demo-cuda --report results/cuda-demo.json + .build/compiler-demo-cuda > results/cuda-demo.stdout + test "$(cat results/cuda-demo.stdout)" = 1460 + python3 scripts/benchmark_cuda.py --repeats 31 --compile-repeats 9 \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --cpu-binary .build/compiler/release/gpu-cpu-liveness \ + --cuda-binary .build/cuda/cuda-liveness \ + --report results/cuda-benchmark.json + python3 scripts/verify_native_cuda.py \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --benchmark-binary .build/compiler/release/gpu-native-benchmark \ + --references .build/compiler/release/references \ + --repeats 21 --compile-repeats 7 --report results/native-cuda-verification.json + ' 2>&1 | tee results/cuda-run.log + + - name: Retain CUDA evidence, including failures + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-compiler-cuda-${{ env.CANDIDATE_SHA }}-${{ github.run_attempt }} + path: | + experiments/gpu-rust-compiler/results/ + experiments/gpu-rust-compiler/.build/verification/cuda/ + experiments/gpu-rust-compiler/.build/compilations/ + experiments/gpu-rust-compiler/.build/benchmarks/cuda/ + experiments/gpu-rust-compiler/.build/verification/native-cuda/ + experiments/gpu-rust-compiler/.build/verification/codegen-cuda/ + experiments/gpu-rust-compiler/.build/verification/codegen-cpu/ + experiments/gpu-rust-compiler/.build/verification/full-rust-cuda/ + experiments/gpu-rust-compiler/.build/nemoclaw-corpus/ + experiments/gpu-rust-compiler/.build/nemoclaw-corpus-packets/ + experiments/gpu-rust-compiler/.build/nemoclaw-corpus-target/fast/*/.gpu-link-receipts/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + - name: Remove the local experiment image + if: always() + working-directory: . + run: docker image rm "$CONTAINER_IMAGE" || true diff --git a/experiments/gpu-rust-compiler/.gitignore b/experiments/gpu-rust-compiler/.gitignore new file mode 100644 index 00000000000..3d842c83a1c --- /dev/null +++ b/experiments/gpu-rust-compiler/.gitignore @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +.build/ +results/ +__pycache__/ +*.pyc +compiler/target/ diff --git a/experiments/gpu-rust-compiler/ARCHITECTURE.md b/experiments/gpu-rust-compiler/ARCHITECTURE.md new file mode 100644 index 00000000000..23e0c95c7a5 --- /dev/null +++ b/experiments/gpu-rust-compiler/ARCHITECTURE.md @@ -0,0 +1,149 @@ + + + +# A new hybrid Rust compiler + +The intended product is a Rust compiler whose CPU workers handle latency, +dependency ordering, and host effects, while GPU workers handle sufficiently +large batches of regular compiler work. CUDA is the primary NVIDIA backend. +Metal implements the same analysis contract for development on Apple Silicon macOS. +The hardware determines the available backend; the compiler's internal +representation and correctness rules should be shared. + +The [compile-time-first backend design](COMPILE_FIRST.md) now supplies a second +path: stock Rust handles the complete language, while a pinned bitcode wrapper +selects CPU or GPU object emission. The scalar frontend described below remains +an independent analysis experiment. Its language restrictions do not apply to +the compatible Cargo path; strict GPU module eligibility still limits GPU +coverage. Multi-GPU scaling and whole-build GPU speedups remain unproved. + +## The vertical slice that exists + +The `compiler/` crate contains a new lexer, parser, scalar type validator, and +LLVM IR emitter. It does not invoke rustc to understand or compile its input. +Rust is used to build the compiler itself. Clang/LLVM generates CPU machine +code from the emitted IR, as an initial target backend. + +The native path now records SSA value IDs, CFG edges, and phi predecessor uses +during emission. Rust packs that compiler-owned graph directly and prunes +instructions using the result. The Metal C ABI adapter caches its library, +pipeline, queue, and reusable shared buffers inside the compiler process. +`--serve` retains this state across requests and recovers from invalid input. +Clang remains an external code-generation/linking process. + +CUDA now has an in-process C ABI adapter with retained streams, events, device +buffers, and pinned staging. Its dense and delta-frontier kernels consume the +same indexed snapshot. Submission captures input, enqueues transfers and work, +and returns before completion; CPU subsets can execute before the final wait. +Contexts and capacity are retained, while facts are cleared and reseeded for +every changed request. The placement policy exposes an explicit unverified +coherent option; capability queries cannot make that option verified. + +The legacy portable `scripts/compile.py` bridge extracts that IR's SSA liveness problem +and selects the independent CPU oracle or the CUDA command-line backend. The selected result feeds a +bounded dead scalar instruction pruning pass, after which Clang links a native +executable. Native tests compare output with rustc compiling the same source +inside an equivalent entry harness. + +Supported source features: `i64` and `bool`, functions and forward calls, +mutable locals, assignments, conditionals, loops, returns, integer operators, +and lazy `&&`/`||`. Source is treated as a module with a zero-argument scalar +`main` function; the generated executable prints that function's result. +This is an experimental entry convention, not complete Rust binary behavior. +Arithmetic uses wrapping add/subtract/multiply and masked shifts. Invalid +division/remainder traps; it does not reproduce Rust panic messages. + +Ownership, references, traits, generic code, pattern matching, macros, +procedural macros, the standard library, dependency crates, Rust metadata, and +Cargo's rustc interface are outside the current subset. Unsupported syntax and +invalid types are rejected. This scalar frontend cannot build NemoClaw or the +full Omarchy Rust tools. + +## Architectures to pursue + +| Design | CPU role | GPU role | First experiment | +|---|---|---|---| +| Adaptive heterogeneous compiler | Parse, validate, schedule dependencies, and execute small or long-chain analyses | Process immutable batches of compact functions | Native CUDA routing requires measured total-cost wins; CPU and GPU state persist | +| Sparse graph engine | Order CPU worklists and create shape buckets | Propagate newly discovered dataflow bits through active frontiers | Delta frontier implemented; SCC preprocessing remains unmeasured | +| Flat task evaluator | Drive host effects, incremental caches, and uneven tasks | Execute balanced independent continuations on persistent workers | Encode one analysis in a Bend-style flat IR and compare against native kernels | +| Compatibility bridge | Use rustc's existing frontend for unsupported Rust while the new frontend grows | Run the same backend-neutral analyses inside a version-pinned backend | A custom `CodegenBackend` adapter with explicit compatibility modes | + +The Metal hybrid is a concrete instance of the first design. It dispatches +eligible functions of at most 64 blocks only when the batch contains at least +32 functions and either 50,000 packed cells or 1,024 word groups. Other functions +remain on CPU worklists. GPU submission precedes CPU work; there is one final +wait and then results are assembled. These provisional thresholds came from the original Metal prototype; the +CUDA routing instead uses calibrated shape buckets and saved CPU worker limits. +Profiles require separate training and held-out total-cost wins and are bound +to hardware, execution objective, and source revision. Unmatched inputs stay +on CPU; admitted GPU batches can run while CPU worklists solve other functions. + +The GPU kernel's words are independent, so a threadgroup can converge without +global GPU synchronization. Long graphs are a poor fit for its dense Jacobi +sweeps: a chain can require roughly N sweeps over N blocks. A reverse-order CPU +worklist may solve that chain with roughly one pass. This algorithmic difference +explains why more GPU cores cannot fix the current stress-case loss. + +For sparse liveness, initialize each block with local uses plus outgoing phi +uses not killed by local definitions. When a successor gains bits, propagate +only those bits not killed by the predecessor. Immutable GEN/KILL data gives a +monotone fixed point. Duplicate-safe atomic updates and correct termination +are required. Changed compiler input must invalidate prior results; deletion +cannot be handled by a monotone add-only update alone. + +[Rust's MIR dataflow framework](https://rustc-dev-guide.rust-lang.org/mir/dataflow.html) +provides actual fixed-point analyses and edge effects to adapt. +[Gunrock](https://arxiv.org/abs/1701.01170) provides the active-frontier graph +architecture; its CUDA implementation is not a Metal implementation. +[Bend's flat evaluator](https://github.com/bendlang/bend/blob/main/guide/GUIDE.md#under-the-hood) +is an alternative execution model requiring substantial compiler data redesign. +[rustc's backend interface](https://github.com/rust-lang/rust/blob/main/compiler/rustc_codegen_ssa/src/traits/backend.rs) +provides a compatibility seam with compiler-version coupling. + +## CUDA and Metal validation + +Both GPU implementations consume the same bitsets and function-word groups, +and solve the same least fixed point. Native CUDA consumes compiler-owned +arrays directly. The legacy CUDA executable and native benchmark driver also +accept `GLC1` packs and emit `GLR1` results for independent verification. The +experimental workflow must establish CUDA compilation and NVIDIA GPU execution +on its exact candidate revision. Metal execution on a Mac cannot provide that evidence. + +On a CUDA-capable build host: + +```sh +python3 scripts/verify_cuda.py --report results/cuda-verification.json +python3 scripts/compile.py fixtures/compiler_demo.rs --backend cuda \ + --output .build/compiler_demo_cuda +``` + +The CUDA driver fails if no compatible device or runtime is available. There +is no silent CPU fallback. It verifies its result against the CPU oracle before +using it for pruning. Hardware results must identify the GPU, toolkit, driver, +source hashes, transfer/setup cost, and total compile time. + +## Growing into a Rust compiler + +1. Establish the scalar source-to-native slice and GPU analysis invariants. + This is implemented; the native and independent oracle tests verify its scope. +2. The native compiler adapter and persistent GPU pipelines are implemented. + Analysis no longer reparses LLVM text or crosses a JSON/subprocess boundary. + Fresh processes and persistent workers are benchmarked against the same + source and standard Rust with an equivalent entry harness. Native CPU analysis can use scoped workers; + CPU execution now also retains a worker pool and calibrates mode and active + worker count. Clang code + generation and linking remain separate subprocess costs. +3. Profile actual NemoClaw and Omarchy-component Rust builds. Select an analysis + with enough aggregate cost to justify GPU work; the current liveness pass + is not evidence that it dominates those builds. +4. Expand source compatibility in deliberate stages: aggregates/control flow, + ownership and borrow checking, generics/traits/coherence/const evaluation, + macros and procedural macros, standard-library and dependency compilation, + Cargo metadata and ABI compatibility. +5. Require differential diagnostics and program-behavior tests before widening + accepted syntax. A compiler accepting more source while changing its meaning + is a regression. Bind performance claims to exact sources and toolchains. + +The immediate performance target is a lower complete compile time with the +same language features and output behavior. A fast isolated pass, simpler +language semantics, or omitting compiler work is insufficient evidence. diff --git a/experiments/gpu-rust-compiler/COMPILE_FIRST.md b/experiments/gpu-rust-compiler/COMPILE_FIRST.md new file mode 100644 index 00000000000..d03b6e20107 --- /dev/null +++ b/experiments/gpu-rust-compiler/COMPILE_FIRST.md @@ -0,0 +1,184 @@ + + + +# Compile time first: CPU and GPU object emission + +This non-shipping experiment preserves Rust's semantic frontend and replaces +object emission at a pinned Rust 1.98.1 / LLVM 22.1.8 boundary. Compile time is +the first objective. Slower generated programs are acceptable in `fast` mode; +`release` rebuilds through the project's optimizing compiler configuration. +Compiler runtime assets and complete generated executables each have a separate +1.5 times size allowance. Physical GPU execution and compilation speedup are +separate claims. + +## Implemented path + +```mermaid +flowchart TD + A[Cargo policy: fast, balanced, release] --> B[Existing Rust semantic frontend] + B --> C[Verified LLVM bitcode and Rust saved link state] + C --> D[TPDE CPU object emission] + C --> E[Strict whole-module GPU eligibility] + C --> F[Explicit LLVM CPU fallback] + E --> G[Metal or CUDA native code bytes] + D --> H[Objects and required metadata] + F --> H + G --> H + H --> I[Existing Rust packaging and linker] +``` + +Rust retains macros, procedural macros, type and trait checks, borrow checking, +const evaluation, layout, monomorphization, crate metadata, and Cargo build +scripts. The wrapper requests bitcode with `linker-plugin-lto`, materializes the +selected objects, and invokes Rust's saved link step. Receipts verify that +native objects did not already exist before the selected emitter and that +linking did not recompile source. A local bootstrap setting enables this pinned +internal seam; an empty allowed-feature list preserves stable source feature +gating. + +The CPU bridge parses serialized input with its own LLVM build; LLVM pointers +never cross library boundaries. TPDE emits Linux ELF for supported inputs. +Unsupported TPDE input and Darwin emission require explicit LLVM fallback. +The compact `rust-llvm-llc` helper uses the compiler's existing shared LLVM +library through its C API. The wrapper passes each crate's actual LLVM +machine-code optimization level, preserving package overrides. The standalone +helper/controller default remains O0; this selection does not add another IR +optimizer pipeline. Development headers and tools belong to validation, +not the runtime budget. See [provenance](fast-backend/PROVENANCE.md). + +The GPU emitter accepts a complete eligible LLVM module: public C ABI functions +with zero or one `i64` argument, wrapping constants/copies/add/subtract/multiply, +and return. Unsupported control flow, calls, layouts, protection attributes, +poison flags, and metadata reject eligibility. It does not relax Rust semantics +to increase coverage. CPU metadata generation preserves symbols and required +unwind information. [GEM1](fast-backend/GEM1.md) defines the checked flat packet. + +Metal and CUDA run output counting and machine-code emission on physical GPUs. +CUDA performs its prefix scan on the device; Metal currently uses a timed CPU +prefix step. Both retain contexts and bounded buffers across batches. Objects +contain the GPU-generated bytes. The independent CPU encoding oracle is included +in the full module emitter's total cost during qualification. + +## Controls and ownership + +| Control | Implemented behavior | +| --- | --- | +| `fast` | Root release profile O1, 16 codegen units, LTO off | +| `balanced` | Root release profile O2, 16 codegen units, LTO off | +| `release` | Project release profile and native backend | +| Object backend | `native`, `tpde`, `llvm-bitcode`, or `gpu` | +| GPU requirement | Explicit Metal/CUDA device and tools; fresh CPU-only fallback fails qualification | +| Fallback | Explicit permission, attributed emitter and rejection reason | +| Generated size | Declared reference, consistent complete-file measurement, failing ratio gate | +| Cache reuse | Tool/policy-owned targets; cached GPU lineage never claims a new GPU execution | + +Profiles preserve panic, overflow, debug, feature, and target settings; inherited +compiler flags and package-specific profiles retain Cargo precedence. Optimization +can affect native build scripts through Cargo's `OPT_LEVEL`, so whole-build gains +are not solely Rust backend improvements. [POLICY.md](POLICY.md) documents commands. + +The build-scoped scheduler validates immutable batches, prerequisites, source +generations, queue/memory budgets, CPU credits, critical-path priority, and +starvation prevention. Cancellation retains buffers until GPU work completes or +teardown fences it. Stable symbol ordering makes object layout independent of +completion order. Auto routing uses CPU until calibrated. + +The scheduler is currently an in-process service. The Cargo wrapper uses a +separate emitter process per eligible codegen unit; it does not yet share one +persistent GPU service across all rustc processes. Cargo jobserver descriptors +are preserved, but a daemon-wide token bridge is not implemented. These costs +and missing pieces matter to subsequent performance measurements. + +## Measurements and qualification + +Actual NemoClaw CPU screens on an Apple M2 Pro used clean, locked offline CLI +builds with separate target directories. Default O3/local ThinLTO samples averaged +172.335 seconds. O1/16/LTO-off samples took 115.46 and 128.69 seconds, averaging +122.075 seconds: an exploratory 29.16% reduction. Their consistently stripped +executable was 30.82% larger. These were separate screening runs, not randomized +matched qualification and not GPU measurements. + +The compatible driver subsequently built and ran NemoClaw with `fast` native +CPU emission. Its stripped executable measured 41,090,816 bytes against the +31,278,848-byte reference, passing the 1.5 limit at 1.3137. Its 160.30-second +Cargo time is a separate observation with a different invocation, not a matched +speedup. The prior SDK runtime screen observed a 25.78% regression, acceptable +to investigate under the fast-mode goal. Report compile-and-test time separately +when slower tests offset compilation savings. O0's complete executable exceeded +the size cap; code-section size alone would conceal that failure. + +The complete NemoClaw CLI also builds through the alternative LLVM-bitcode +emission seam: 3,536 translated LLVM object units and 25 attributed passthrough +invocations, including five expected failed empty probes and no failed semantic +invocations. The generated CLI executes `--version`. Matching each crate's LLVM +code-generation level produces a 41,319,680-byte stripped executable, 32.10% +above the reference and within the limit. Cargo elapsed 193.536 seconds and total +driver time 199.692 seconds; no speedup is established. The earlier forced-O0 +variant compiled and ran but produced 55,215,120 bytes (+76.5%) and correctly +failed the size gate. Both observations retain their separate reports. + +Separate Mac runtime-file accounting measures 235,129,353 bytes for the compiler +and required adapters/kernels, against the 222,966,640-byte pinned compiler +baseline: +5.455%, within the separate compiler allowance. It includes the +actually linked Metal libraries and counts shared LLVM once. Unchanged Cargo, +standard libraries, system frameworks, caches, benchmarks, and development tools +are outside this compiler-runtime comparison. Linux/CUDA asset accounting is +still separate qualification work; the bridge statically links its needed LLVM +components to avoid another complete shared LLVM runtime. + +On this Mac, the full Rust fixture links a math crate emitted by physical Metal +with attributed CPU objects for other modules. It executes boundary arithmetic, +generics, standard-library code, caught panic, and drop checks. An identical +second build must reuse qualified lineage while reporting no new GPU execution. +The verifier checks raw bitcode/emitter/saved-link evidence. This proves the +exercised integration, not arbitrary Rust GPU coverage or a performance win. + +The draft PR workflow builds complete NemoClaw under native CPU release/fast +policies, compiles the CUDA emitter, compares CPU/GPU leaf batches on the same +runner, and verifies the full Rust CUDA fixture plus cache behavior. It retains +source/tool/device identity, raw receipts, objects, outputs, and costs. Only a +successful workflow on the exact current head qualifies that revision's NVIDIA +execution; Mac Metal evidence cannot qualify CUDA. + +## Scaling target and next gates + +Keep quality fixed and distribute independent concrete functions or modules +across devices. `gpu-codegen --devices 0,1,...` creates physical CUDA contexts and +dispatches independent batches. Simulated devices test scheduling only. A +one-GPU runner cannot establish a two- or four-GPU scaling curve. + +For a first model, `T(N) = F + L + P + I(N) + G/(N*e(N)) + tail(N)`. +`F` is dependency-critical frontend/host work, `L` is object/link work, `P` is +unoverlapped preparation, `I` is initialization/transfers/shared host capacity, +and `G` is useful single-GPU work. Efficiency and imbalance must be measured. +For overlapping tasks, use the critical path instead of a sum of thread durations. +An ideal fixed share `s` bounds speedup by `1/(s+(1-s)/N)`; more GPUs cannot +remove frontend, preparation, or linking bottlenecks. + +Near-linear throughput is more plausible across many ready, unique modules/builds +until frontend feed, staging, or linking saturates. Strong scaling holds the +build and CPU resources fixed; throughput studies disclose added host resources. +More optimization search at fixed release-time budget is a separate quality +experiment, not proof of faster compilation. + +The final bounded test first captures every module from a complete fast build +and verifies whole-module GPU eligibility. The local Mac corpus contains 3,536 +modules and zero eligible modules; it gives this implementation no available +GPU work. The workflow repeats the audit on the NVIDIA runner. Nonzero eligible +work enters persistent parallel CPU/GPU replay before any complete-build +qualification. The continuation gate requires a repeatable improvement of at +least 10% over the strongest compatible CPU fast build, including preparation, +transfers, verification, metadata, fallback, linking and both size limits. +Zero eligibility parks this GPU route. See [the gate](FINAL_GPU_GATE.md). + +Persistent CPU emission and full Linux TPDE qualification remain research leads. +The bounded GPU test does not authorize a Cargo-wide daemon, expanded Rust +semantics or multi-GPU work. Physical 1/2/4-GPU measurements would follow a +qualified single-GPU complete-build win. + +The CPU lead is [TPDE's CGO 2026 paper](https://aengelke.net/pubs/2602-cgo1.pdf): +its reported 8–26 times backend improvement and 9–24% C/C++ whole-build reductions +motivate qualification, not a Rust claim. The GPU layout draws on +[Pareas's count/scan/emission design](https://theses.liacs.nl/pdf/2020-2021-HuijbenMarcel.pdf). +The compatible CPU route leaves usable compile-time controls if the GPU hypothesis +fails. This draft remains excluded from production and must not be merged. diff --git a/experiments/gpu-rust-compiler/FINAL_GPU_GATE.md b/experiments/gpu-rust-compiler/FINAL_GPU_GATE.md new file mode 100644 index 00000000000..c7c6135b015 --- /dev/null +++ b/experiments/gpu-rust-compiler/FINAL_GPU_GATE.md @@ -0,0 +1,84 @@ + + + +# Final bounded GPU test + +Continue this GPU route only after a repeatable reduction of at least 10% in +complete fast-build time against the strongest compatible CPU baseline, with +semantic checks and separate 1.5-times compiler/program size limits passing. +Kernel timing, synthetic throughput and cached builds cannot pass this gate. +This draft must remain non-shipping. + +## Available compiler work + +First capture every LLVM module from a fresh complete NemoClaw fast build. +Set `GPU_LINK_CAPTURE_DIR` on a `gpu-cargo build` using the `llvm-bitcode` +backend and a new target root. The wrapper saves exact bitcode before replacing +it with a CPU object. Each invocation owns a separate directory and manifest. +Capture copies, hashes and reports are included in the audit's costs; its build +time is not a qualified performance comparison. + +`gpu-corpus-census` inspects every captured module through the pinned bridge's +strict exporter. It verifies input bytes and matches manifests, arguments, +targets, code-generation levels, costs and input-hash multiplicities to completed +wrapper receipts. Capture counts must equal the build's translated-unit count. +Accepted packets also pass the CPU encoding and native object contract. +Invalid input or broken tooling fails the census instead of counting as +unsupported work. + +The October 10 Mac audit captured **3,536 modules across 494 translated Rust +invocations**. **Zero whole modules qualified**. Rejection reasons included: + +| First rejection | Modules | +| --- | ---: | +| Globals, aliases, indirect functions or module assembly | 2,615 | +| Unsupported C ABI/signature | 726 | +| Unsupported linkage/visibility | 102 | +| No defined functions | 61 | +| Unsupported argument type | 22 | +| Export input exceeds the current 2 MiB bound | 6 | +| Other unsupported attributes, control flow or arithmetic | 4 | + +These are first rejection reasons, not a complete census of every construct +inside each module. The recorded checkout was dirty; this is local Mac evidence, +not exact-head Linux/CUDA qualification. The generated NemoClaw executable ran +`--version` and measured 41,319,680 bytes after consistent stripping, versus +31,278,848 for its declared reference (+32.10%, within the program limit). + +A separate Rust control fixture produced one eligible module and seven rejected +modules. It exercised capture, census and five alternating persistent CPU/Metal +replay rounds with stable object bytes. A further two-module hardware test +checks parallel CPU workers and retained Metal sessions. These are validation +fixtures, not larger NemoClaw workloads or evidence of a compilation speedup. + +## Replay and decision + +For nonzero eligibility, `gpu-corpus-replay` consumes the census directly. +It verifies snapshots and packet hashes, deduplicates identical inputs, retains +parallel CPU and physical GPU services, and alternates complete corpus rounds. +Each unique module runs once per side per round; repeats do not create additional +distinct work. It records decode/pack, queues, device execution, independent CPU +byte checks and object serialization. Worker durations overlap and must not be +added as whole-build wall time. GPU-only eligibility work is charged separately +to the GPU route. + +The leaf CPU encoder is a matched algorithm comparison. It does not qualify the +strongest complete Rust CPU backend. Replay excludes Rust frontend, unsupported +modules, fallback, linking and generated-program runtime. Stable objects do not +establish native program execution. A favorable replay would only justify a +subsequent matched complete-build comparison. + +If no whole modules qualify, available work for this GPU implementation is +exactly zero, even with a zero-cost GPU. Park this route without manufacturing +a larger batch or implementing a daemon. Supporting arbitrary signatures, +globals and control flow would require another compiler project and falls +outside this final bounded test. Preserve CPU compile-time controls and the +experiment's evidence. No multi-GPU scaling claim is established. + +The draft workflow repeats the complete capture and census in its offline CUDA +validation environment on the RTX PRO 6000 runner. Locked dependencies are +fetched beforehand, and protoc matches the repository's pinned version. +Only eligible captured work enters CUDA replay. It retains exact-head source +identity, raw bitcode, manifests, receipts and reports, including failures. +The independent CUDA integration checks still verify GPU object emission and +full Rust fixture linking/cache behavior. diff --git a/experiments/gpu-rust-compiler/IMPORT.md b/experiments/gpu-rust-compiler/IMPORT.md new file mode 100644 index 00000000000..27344dbdfe5 --- /dev/null +++ b/experiments/gpu-rust-compiler/IMPORT.md @@ -0,0 +1,51 @@ + + + +# Prototype import + +Imported from the local `gpu-rustc-lab` prototype developed for this experiment. +The source has no published repository revision; the hashes below identify the +original files before this PR adapted them. The originals contain no third-party +copyright notices. This import adds the repository SPDX notices. + +Changes: isolate the Cargo workspace; retain the native frontend and Metal adapter; +remove workstation capture and downloaded tool benchmarks; replace the CUDA +bridge's Swift dependency with the portable set oracle; allow an explicit frontend +artifact; limit bridge choices to CPU/CUDA; remove the legacy driver from the +native benchmark; make existing adapter tests run CPU checks on Linux. + +No generated binaries, build trees, personal paths, or prior benchmark results +are imported. The compiler frontend, liveness kernels, and code generation semantics +are unchanged. + +| Original file | SHA-256 | +|---|---| +| `.gitignore` | `ffb551f742928f3156f648ef4053b6f77c124a078e5ce2062fee7369c6a4278e` | +| `ARCHITECTURE.md` | `233b396b064b76eb31090fe9e5c9c25288c0f9d56f6226fa0c5a88ea1fa1c4c5` | +| `README.md` | `817648a074510fe23af1ef09cd1dfe86ad2ad35e0b859d4da94465f4023d4996` | +| `compiler/Cargo.lock` | `1a049088308b5ce9c13cbca60eecbbd06c7976365e503343aebee65525aee7df` | +| `compiler/Cargo.toml` | `cf766cde51b1b8be89b15ea6ecfa7c38e3696d4ecc2ce6602c5805d41952afe4` | +| `compiler/build.rs` | `732701160e6ed7c8e82fe6e77d81be94f5bfcbae30ce59f9102531a81ec30464` | +| `compiler/src/analysis.rs` | `01c1b6ef8730b7dd41819f5a6af1e6d4639e2f13482ed1c4d700363bf7814a3f` | +| `compiler/src/frontend.rs` | `2b491a0b2ede8812856b16a0c6203df4f0bc0c497bcf205eb3ba3e7d65088473` | +| `compiler/src/ir.rs` | `c1e204c305e1e8727fa1ed4f10eca9c6a2a6ceccc88ff6203a6eba33df0b755c` | +| `compiler/src/llvm.rs` | `ddd7804f5c960d07c6a5f801a0817cb05dcf00ab541d1be2c06d71362d27aa9b` | +| `compiler/src/main.rs` | `96a9ede287b8007e9f7250e3484bbe816f9f0a3485ac0860dd3958b97c84ed20` | +| `compiler/src/metal.rs` | `9fb57049d6976ee0f5c140d496009b3328332cbaca0b2b9fde8c42ac6cdf46a4` | +| `cuda/CMakeLists.txt` | `5b839e40cf3eb2fc58d9af6b73f6bdb3fd350c38eba6547aa2c538ad57de962b` | +| `cuda/liveness.cu` | `6a58f20414317684416920d8352185117308f677fd6954a2e8a3247a82b01e60` | +| `fixtures/compiler_demo.rs` | `cf5d7480a466dbc181314284d36c30324cdd2bf6b8a0b1717651617c93a497b0` | +| `native/metal_bridge.h` | `523231b12386b5479ab1947eff8679fbe4715530fb3286e54fe5f1b1087ddd25` | +| `native/metal_bridge.mm` | `dd318c71f8a0654f88a6e21a3165f266a769821cd21e1943ae3f1a37aa250fb2` | +| `scripts/benchmark_native.py` | `b62e2dcfe3fde236f64cd4b583a0d57fcdbc9e4de674186918799e410765c86e` | +| `scripts/build_native.sh` | `c56b7e90e07c5b75e951d914ff23f043ab15020ef1f817857ea3a8f7b353b935` | +| `scripts/compile.py` | `0e4a5c0ef482bf956f80b4b9fbf6f8c04bd7aa569c88e1a0717071075c171e9d` | +| `scripts/export_cuda.py` | `e869d19956a01e5b11ce21ba74abef0aaa03f7fec308231462cfe67c75298fd7` | +| `scripts/liveness_reference.py` | `467e2361f8cc02a76ac916ec29a136ba8b319c7618ce47ed4c99a713c0be3721` | +| `scripts/optimize_ir.py` | `c45135cf1ca0c6f54148fcc6a61c7159bc583d0c8767d7845a519bd403e19715` | +| `scripts/prepare_workload.py` | `6fa899a24b7c1d3ea983ebeb2fd9bfd9f2d289ebe317ebaa6cba4a7797da818c` | +| `scripts/verify_cuda.py` | `1493ae34cbc6bbef4378cfe9bc3fc14492424ff77095985fd8bfe87da017ef35` | +| `src/liveness.metal` | `38e94e0e0fa8ebaeb1a86cce4b3eb622b5e4397af40e8b91027cc97d1aee9770` | +| `tests/test_liveness_reference.py` | `80edac08b9985f228ed272fb955bdeab4891a1cd94781fcc7ac3613797561520` | +| `tests/test_native_adapter.py` | `7cee490374c094e39bbfc9c3032dd3226149228b826bef1770cff569a9fc524a` | +| `tests/test_prepare_workload.py` | `0ebcfc8922eacaad3a141a48302a0a1c39166092cdc80e919d864645c6adeda0` | diff --git a/experiments/gpu-rust-compiler/POLICY.md b/experiments/gpu-rust-compiler/POLICY.md new file mode 100644 index 00000000000..1d4d91e47cb --- /dev/null +++ b/experiments/gpu-rust-compiler/POLICY.md @@ -0,0 +1,144 @@ + + + +# Compatible compilation policies + +`gpu-cargo` builds Cargo packages with pinned Rust 1.98.1. The default `native` +object backend uses stock CPU LLVM. Experimental `tpde`, `llvm-bitcode`, and +`gpu` adapters reuse Rust's frontend, crate metadata, packaging, and linker. +GPU object emission is restricted to qualified modules; other modules require +explicit, attributed CPU fallback. Physical execution does not establish a +compilation speedup. + +| Quality | Root release-profile overrides | +| --- | --- | +| `fast` | O1, 16 codegen units, LTO off | +| `balanced` | O2, 16 codegen units, LTO off | +| `release` | Existing Cargo release configuration | + +Every quality uses Cargo's release profile. The driver preserves the project's +panic, overflow, debug, target, feature, and inherited semantic settings. Existing +package-specific profile overrides and inherited compiler flags retain Cargo +precedence; the report identifies this scope. It pins Cargo's actual rustc path +and selects an explicit adapter wrapper, disabling project wrappers. It invokes programs as argument vectors and +retains the inherited environment and descriptors for Cargo's jobserver. + +Run from the repository root: + +```sh +cargo +1.98.1 build --locked \ + --manifest-path experiments/gpu-rust-compiler/compiler/Cargo.toml \ + --target-dir experiments/gpu-rust-compiler/.build/compiler \ + --bin gpu-cargo + +experiments/gpu-rust-compiler/.build/compiler/debug/gpu-cargo plan \ + --manifest-path Cargo.toml --package nemoclaw-cli --bin nemoclaw \ + --quality fast --placement auto --offline \ + --target-root experiments/gpu-rust-compiler/.build/cargo-policies \ + --report experiments/gpu-rust-compiler/results/cargo-plan.json +``` + +Replace `plan` with `build` to compile. `--dry-run` also selects planning. Plans +inspect the toolchain and locked Cargo metadata but do not build or create target +directories. Required `metal` or `cuda` placement needs `--object-backend gpu` +and explicit GPU tools. The other object backends reject required GPU placement. + +The selected target root contains separate `fast`, `balanced`, and `release` +directories, with a `native` child or a backend/tool-provenance child. The driver creates an ownership marker in each. It refuses to adopt +a nonempty directory without its marker or reuse a directory belonging to +another manifest or policy. Existing build targets remain separate. + +## Experimental object adapters + +Alternate object backends support `fast` and `balanced`. `release` retains the +native Cargo LTO contract. Supply all four paths: `--bitcode-wrapper`, +`--tpde-controller`, `--tpde-bridge`, and `--llvm-llc`. The controller uses the +pinned bridge to inspect IR even when LLVM emits the object. See +[backend setup](fast-backend/README.md). + +`--object-backend llvm-bitcode` qualifies the bitcode/packaging seam using LLVM. +`--object-backend tpde` requests TPDE; `--allow-backend-fallback` permits recorded +LLVM fallback. TPDE cannot emit Darwin objects in the current adapter, so this +Mac needs that explicit fallback. Reports name the actual emitter rather than +labeling an LLVM result as TPDE. + +`--object-backend gpu` also requires `--gpu-emitter` and `--gpu-library`. Select +`--gpu-backend metal|cuda`, optionally `--gpu-device INDEX`; Metal requires +`--gpu-shader`. A fresh GPU build must contain at least one physical GPU object +unit with recorded device identity, kernel dispatches, and used kernel output. +CPU fallback and module eligibility failures are reported separately. These +adapters do not establish arbitrary Rust-module GPU coverage or a performance +win. + +Each Cargo invocation owns a new receipt directory. Reports aggregate current +object emitters, fallback reasons, and frontend/materialization/emission/link +stages. Stage sums can overlap across compiler processes; they are not +whole-build wall-time percentages. Tool-content hashes and fallback/device +choices isolate backend targets. + +LLVM object emission follows each actual rustc invocation's code-generation +optimization setting, including package overrides and inherited flags. Size +levels use the pinned compiler's target-machine mapping and retain IR size +attributes. The standalone controller/helper keep O0 as their default baseline; +`--llvm-codegen-opt-level 0|1|2|3` selects only machine-code generation. + +For the final coverage audit, set `GPU_LINK_CAPTURE_DIR` to an unused corpus +root while building `llvm-bitcode` into a fresh target root. The wrapper retains +each exact frontend bitcode module and its manifest before replacing the private +input with a native object. Capture costs remain in the audit's timings; it +cannot establish a performance win. `gpu-corpus-census --help` and +`gpu-corpus-replay --help` describe the capture verification and persistent +replay commands. [The final gate](FINAL_GPU_GATE.md) defines the stopping rule. + +Alternate modes reject an inherited `RUST_LLVM_LIBRARY` override. They resolve +and hash the pinned sysroot library, explicitly select it only for child Cargo, +and verify the compact helper's loaded-library report. External `llc` remains +usable, but its additional dynamic dependencies are unqualified; it cannot +establish reusable GPU cache lineage. + +A successful unchanged GPU build can reuse an owned, qualified artifact with +matching input/tool/artifact hashes and unchanged prior receipt hashes. It +reports `cache_only=true`, `gpu_executed=false`, and +`current_run_gpu_qualified=false`, retaining prior lineage separately. Cached +work cannot serve as evidence of a new GPU run. A fresh CPU-only fallback build +fails required-GPU qualification and invalidates previous GPU artifact lineage. + +## Generated executable size + +Declare a reference executable outside the selected quality's target directory: + +```sh +experiments/gpu-rust-compiler/.build/compiler/debug/gpu-cargo build \ + --manifest-path Cargo.toml --package nemoclaw-cli --bin nemoclaw \ + --quality fast --offline \ + --target-root experiments/gpu-rust-compiler/.build/cargo-policies \ + --reference-artifact /absolute/path/to/reference-nemoclaw \ + --max-size-ratio 1.5 --size-basis strip-debug \ + --report experiments/gpu-rust-compiler/results/cargo-fast.json +``` + +`unmodified` measures each complete original file. `strip-debug` copies both +files and applies the same host strip command to each before measuring the +complete copies; the originals remain unchanged. On macOS it uses `strip -S -x`; +on Linux, `strip --strip-debug --discard-all`. This is complete-file accounting, +not machine-code-section accounting. The report retains raw sizes, measurement +method, hashes, and the observed ratio. + +The size gate has no baseline until `--reference-artifact` is supplied. An +executable exceeding the declared ratio returns a failing exit status and a +`size-limit-exceeded` report. It does not silently rebuild or alter semantics to +fit the limit. Compiler footprint and its separate +50% allowance require +additional accounting; this generated-executable gate does not qualify them. + +JSON reports include the exact command vector, pinned compiler path/version, +manifest and lock hashes, available Git revision/dirty state, actual backend, +actual Cargo executable path, compilation time, total driver time, artifact +size/hash, and optional size comparison. Compilation time includes Cargo's +selected dependency/build-script/link work, including cache hits. Source +provenance does not claim a content hash of every input. Report files replace +only earlier `gpu-cargo-report` version 1 files; unrelated existing files are +preserved. + +Use identical explicit flags and cache states for repeated benchmarks. Switching +between implicit default codegen units and explicit `16` can change Cargo +fingerprints even when the effective codegen count matches. diff --git a/experiments/gpu-rust-compiler/README.md b/experiments/gpu-rust-compiler/README.md new file mode 100644 index 00000000000..21c4c4851c8 --- /dev/null +++ b/experiments/gpu-rust-compiler/README.md @@ -0,0 +1,195 @@ + + + +# GPU-assisted Rust compiler experiment + +This is a disposable pipeline experiment. It is excluded from NemoClaw's +production Cargo workspace and has `publish = false`. Do not merge or ship it. +[Import provenance](IMPORT.md) records the original prototype hashes. + +The compile-time-first path now builds complete Rust programs through the +existing Rust frontend and a replaceable object emitter. `gpu-cargo` selects +`fast`, `balanced`, or `release` quality independently of the backend. The +experimental bitcode wrapper replaces object emission, then uses Rust's saved +link step without compiling the source again. CPU routes use TPDE or an explicit +LLVM fallback; GPU routes emit native machine code for strict leaf modules. +The full Rust fixture exercises generics, standard-library linkage, boundary +arithmetic, caught panic, and drops with physical GPU code plus attributed CPU +fallback. See [the current design and evidence](COMPILE_FIRST.md), +[policy controls](POLICY.md), and [backend setup](fast-backend/README.md). +GPU execution does not yet establish a faster whole Rust build. + +The earlier independent frontend compiles a scalar Rust subset through SSA liveness analysis, +dead instruction pruning, and Clang/LLVM machine-code generation. Supported +source has `i64`/`bool`, functions, mutable locals, conditionals, loops, returns, +and scalar operators. The executable prints its zero-argument scalar `main` +result. It does not support ownership, traits, generics, macros, dependencies, +the standard library, or Cargo's compiler interface. That scalar frontend cannot build NemoClaw +or the complete Rust components used by Omarchy. + +The native CPU, Metal, and CUDA paths retain their execution state across +`--serve` requests. CUDA loads a small C ABI library in the compiler process; +analysis uses the compiler's indexed arrays directly. Python remains an +independent oracle and benchmark controller inside the validation image. The +older Python/CUDA-process bridge remains a comparison baseline. An explicitly +requested CUDA pass fails when unavailable. Adaptive routing can select CPU +and reports that decision. A faster pass does not establish a faster NemoClaw build. + +## Run the GPU experiment in GitHub Actions + +Keep [PR #12951](https://github.com/NVIDIA/NemoClaw/pull/12951) open as a draft +from `experiment/gpu-rust-compiler` into `v1`, then push a signed commit to that +NVIDIA-owned branch. That starts `.github/workflows/gpu-compiler-experiment.yml`; +rerun its Actions run to repeat the current head. The hosted CPU job first +requires that the draft PR's repository, branches, and head SHA match this push. +NVIDIA's GPU runner rejects `pull_request` events, so this experiment uses the +trusted branch push route. Manual dispatch is unavailable until a workflow +exists on the default branch, and this experiment will not be merged. +The workflow builds the native frontend on a hosted Linux runner, then verifies +CUDA inside its validation image on the repository's GPU runner. It +retains device/toolchain identity, source and binary hashes, correctness +results, timings, and program output as artifacts. +The hosted job also compiles standard-Rust reference programs; their sources, +executable outputs, compiler identity, and checksums travel with this exact-head +artifact. The GPU job builds the native CUDA library from the same revision. + +GPU verification requires that workflow to succeed on this PR's exact +head. Exit code `3` from the verifier means CUDA is unavailable; it is not a +passing GPU result. The generated liveness cases are compiler analysis inputs, +not representative NemoClaw build measurements. + +Within the validation image, from this directory: + +```sh +python3 scripts/verify_cuda.py --report results/cuda-verification.json +python3 scripts/compile.py fixtures/compiler_demo.rs --backend cuda \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --cuda-binary .build/cuda/cuda-liveness \ + --output .build/compiler_demo_cuda --report results/cuda-demo.json +.build/compiler_demo_cuda +# 1460 +``` + +`--compiler-binary` accepts a frontend artifact from the same source revision +without rebuilding it. Without that option, the bridge builds the frontend +when absent or older than its sources. A CUDA device, compatible driver/toolkit, +Python 3.9+, and a host C++ compiler are required. Clang must support the emitted +opaque-pointer LLVM IR. + +## Persistent native CUDA + +`native/cuda_bridge.{h,cu}` exposes creation, submission, completion, device +capability queries, and destruction. A context owns a nonblocking stream, +events, reusable device allocations, and pinned staging buffers. Submission +captures an immutable snapshot before returning; the caller can then perform +independent CPU work. Completion waits for result downloads. Every request +reseeds its facts, including requests that remove previously live values. + +The dense kernel remains a reference. The delta-frontier kernel propagates new +bits backward through predecessor kill masks, using atomic updates and uniform +round barriers. It scans a frontier bitmap but only visits edges of active +rows. Function-size buckets avoid charging small functions for the largest +function's shared-memory allocation. Both algorithms implement the same +finite least fixed point and phi predecessor semantics. + +Inside the CUDA image, the native verifier builds the adapter and runs: + +```sh +python3 scripts/verify_native_cuda.py \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --benchmark-binary .build/compiler/release/gpu-native-benchmark \ + --references .build/compiler/release/references \ + --report results/native-cuda-verification.json +.build/compiler/release/gpu-rust-compiler --serve --backend cuda \ + --cuda-algorithm sparse --cuda-library .build/cuda/libgpulab_cuda.so --verify +``` + +Native `cpu` execution has a reusable worker pool. Analysis calibration compares +serial, function, and word worklists over several active worker counts. CUDA +`cuda-hybrid` routing uses a saved profile bound to device, driver/runtime, CPU +capacity, source revision, objective, candidate shape, and parent workload. +Admission requires a clear timing advantage in separate training and held-out +samples, including packing, staging, transfers, CPU remainder, and assembly. +Missing, mismatched, or noisy evidence selects CPU. Long straight chains stay +on CPU. `--mode latency|throughput` selects separately qualified policies; +latency evidence cannot authorize throughput placement. + +The verifier compares fresh and persistent processes, dense and sparse +algorithms, full supported-program compilation, and interleaved concurrent +compilation cohorts. Held-out facts rotate value identities on the same CFG; +they do not establish generalization to unseen graph geometry. Event polling +checks whether GPU work remains pending when independent CPU work starts; +it does not measure simultaneous kernel/CPU execution duration. + +Placement is separate from algorithm selection. Pinned staging is implemented +on the PCIe runner. Runtime queries report memory capabilities, while +`CoherentUnverified` refuses execution. Vera/Rubin coherent placement and SCC +preprocessing remain unverified until hardware or measured benefits justify them. + +## Same-runner CUDA benchmarks + +The GPU job benchmarks the same generated analysis inputs with the native Rust +CPU worklist and CUDA. CPU results include serial, function-parallel, and +word-parallel solvers; the comparison uses the fastest CPU median. The Python +set oracle checks outputs outside the timed solve and is not the performance +baseline. Compact function batches test GPU throughput at several sizes; +long chains and mixed workloads test propagation and scheduling costs. + +CUDA reports kernel events, resident-input warm passes, and warm passes that +upload all input arrays and download results. The last comparison includes +transfers and retains device allocations. Initialization and input preparation +are reported separately. These generated workloads are analysis measurements, +not NemoClaw or Omarchy build measurements. + +The complete compiler benchmark interleaves repeated fresh-process compiles +of the supported demo through the native CPU compiler, Python CPU bridge, and +CUDA bridge. Each executable must print `1460`. Wall time includes process +startup and linking; stage timings retain their own boundaries. Native CPU +versus CUDA compares the current architectures, including the bridge and CPU +verification costs. Python CPU versus CUDA uses the same frontend and bridge. +File caches and the NVIDIA driver may remain warm between samples. + +Within the validation image, after CUDA verification: + +```sh +python3 scripts/benchmark_cuda.py --repeats 31 --compile-repeats 9 \ + --compiler-binary .build/compiler/release/gpu-rust-compiler \ + --cpu-binary .build/compiler/release/gpu-cpu-liveness \ + --cuda-binary .build/cuda/cuda-liveness --report results/cuda-benchmark.json +``` + +The workflow retains raw samples, workload hashes, executable outputs, hardware +identity, and source provenance. A result for one pass or this scalar demo does +not establish a faster complete Rust build. + +## CPU and Metal development + +```sh +sh scripts/build_native.sh +cargo test --locked --manifest-path compiler/Cargo.toml \ + --target-dir .build/compiler +python3 -m unittest discover -s tests +.build/compiler/release/gpu-rust-compiler fixtures/compiler_demo.rs \ + --backend cpu --verify --output .build/compiler_demo +.build/compiler_demo +python3 scripts/benchmark_native.py --backends cpu --repeats 5 +``` + +CPU checks need Rust, Python, and Clang. The native tests compare emitted +program behavior against standard Rust compiling the same module with an +equivalent entry harness. Metal tests skip on Linux. On Apple Silicon macOS +with Xcode tools, use `--backend metal` or `--backend hybrid`; the default native +benchmark compares all three. Metal work executes on the Mac GPU, which cannot +verify CUDA hardware execution. + +`--serve --backend metal --verify` keeps the Metal pipeline and buffers alive. +Each request is `compile`, source, output, and optional report, separated by +tabs. Each response is one JSON line. Paths cannot contain tabs or newlines. +Clang/linking remains a subprocess for every request. + +Generated data and binaries live in `.build/`; reports live in `results/`. +Both are ignored. Timings separate warm kernel execution from startup, +transfers, verification, parsing, code generation, and linking. The native +benchmark measures complete supported-subset compilation, including linking, +with standard Rust as a behavioral baseline. This experiment has no shipped +performance claim. [Architecture and next milestones](ARCHITECTURE.md). diff --git a/experiments/gpu-rust-compiler/compiler/Cargo.lock b/experiments/gpu-rust-compiler/compiler/Cargo.lock new file mode 100644 index 00000000000..b4fb4f6b222 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/Cargo.lock @@ -0,0 +1,249 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "gpu-rust-compiler" +version = "0.1.0" +dependencies = [ + "object", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "crc32fast", + "hashbrown 0.15.5", + "indexmap", + "memchr", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/experiments/gpu-rust-compiler/compiler/Cargo.toml b/experiments/gpu-rust-compiler/compiler/Cargo.toml new file mode 100644 index 00000000000..b15fff75b46 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/Cargo.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "gpu-rust-compiler" +version = "0.1.0" +edition = "2021" +default-run = "gpu-rust-compiler" +publish = false +license = "Apache-2.0" + +# This experiment is independent of NemoClaw's production workspace. +[workspace] +resolver = "2" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.11" +object = { version = "0.37", default-features = false, features = ["write", "read", "std"] } + +[profile.release] +opt-level = 3 diff --git a/experiments/gpu-rust-compiler/compiler/build.rs b/experiments/gpu-rust-compiler/compiler/build.rs new file mode 100644 index 00000000000..025e4322c81 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/build.rs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{env, path::PathBuf, process::Command}; + +fn main() { + if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") { + return; + } + let root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()) + .parent() + .unwrap() + .to_owned(); + let source = root.join("native/metal_bridge.mm"); + let header = root.join("native/metal_bridge.h"); + println!("cargo:rerun-if-changed={}", source.display()); + println!("cargo:rerun-if-changed={}", header.display()); + let out = PathBuf::from(env::var("OUT_DIR").unwrap()); + let library = out.join("libgpulab_metal.dylib"); + let status = Command::new("xcrun") + .args(["clang++", "-std=c++17", "-O3", "-fobjc-arc", "-dynamiclib"]) + .arg(&source) + .args([ + "-framework", + "Metal", + "-framework", + "Foundation", + "-Wl,-install_name,@rpath/libgpulab_metal.dylib", + "-o", + ]) + .arg(&library) + .status() + .expect("Could not launch the Xcode C++ compiler"); + assert!(status.success(), "Native Metal adapter compilation failed"); + println!("cargo:rustc-link-search=native={}", out.display()); + println!("cargo:rustc-link-lib=dylib=gpulab_metal"); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", out.display()); + let codegen_source = root.join("native/codegen_metal.mm"); + for file in [ + &codegen_source, + &root.join("native/codegen_bridge.h"), + &root.join("native/codegen.metal"), + ] { + println!("cargo:rerun-if-changed={}", file.display()); + } + let codegen_library = out.join("libgpuemit_metal.dylib"); + let status = Command::new("xcrun") + .args([ + "clang++", + "-std=c++17", + "-O3", + "-Wall", + "-Wextra", + "-Werror", + "-fobjc-arc", + "-dynamiclib", + ]) + .arg(codegen_source) + .args(["-framework", "Metal", "-framework", "Foundation", "-o"]) + .arg(&codegen_library) + .status() + .expect("Could not build Metal machine-code adapter"); + assert!( + status.success(), + "Metal machine-code adapter compilation failed" + ); + println!( + "cargo:rustc-env=GPUEMIT_METAL_LIBRARY={}", + codegen_library.display() + ); +} diff --git a/experiments/gpu-rust-compiler/compiler/src/analysis.rs b/experiments/gpu-rust-compiler/compiler/src/analysis.rs new file mode 100644 index 00000000000..a75f2331984 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/analysis.rs @@ -0,0 +1,970 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Native, packed block liveness and conservative scalar dead-code pruning. +//! +//! The frontend supplies value IDs and CFG edges directly. No textual LLVM +//! parsing, subprocess, or JSON round trip participates in this pass. + +use crate::ir::ModuleIR; +use std::{ + collections::HashSet, + ops::Range, + sync::atomic::{AtomicUsize, Ordering}, +}; + +const MAX_CELLS: usize = 100_000_000; +const MAX_VALUES: usize = 1_000_000; + +/// This layout is the five `uint` fields consumed by `liveness.metal`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub struct GPUGroup { + pub block_base: u32, + pub block_count: u32, + pub row_base: u32, + pub words: u32, + pub word: u32, +} + +#[derive(Clone, Debug)] +pub struct FunctionDesc { + /// Index in the original ModuleIR, also retained by subset packing. + pub module_index: usize, + pub block_base: usize, + pub block_count: usize, + pub row_base: usize, + pub words: usize, + pub value_count: usize, +} + +#[derive(Clone, Debug)] +pub struct PackedAnalysis { + pub functions: Vec, + pub groups: Vec, + /// CSR edges use global packed block indices. + pub successor_offsets: Vec, + pub successors: Vec, + pub predecessor_offsets: Vec, + pub predecessors: Vec, + /// One offset per block, followed by the total cell count. + pub row_offsets: Vec, + pub uses: Vec, + pub defs: Vec, + /// Phi incoming values are live out of their predecessor, not live in + /// to the phi's block. The predecessor DEF is applied by the solver. + pub phi_out: Vec, + pub total_cells: usize, +} + +impl PackedAnalysis { + pub fn block_count(&self) -> usize { + self.row_offsets.len() - 1 + } + + pub fn max_blocks(&self) -> usize { + self.functions + .iter() + .map(|f| f.block_count) + .max() + .unwrap_or(0) + } + + fn successor_range(&self, block: usize) -> Range { + self.successor_offsets[block] as usize..self.successor_offsets[block + 1] as usize + } + + fn predecessor_range(&self, block: usize) -> Range { + self.predecessor_offsets[block] as usize..self.predecessor_offsets[block + 1] as usize + } +} + +pub fn prepare(module: &ModuleIR) -> Result { + prepare_subset(module, &(0..module.functions.len()).collect::>()) +} + +/// Subset packing preserves original function IDs so CPU and GPU results can +/// be scattered back into one complete solution without examining LLVM text. +pub fn prepare_subset(module: &ModuleIR, indices: &[usize]) -> Result { + let mut packed = PackedAnalysis { + functions: Vec::with_capacity(indices.len()), + groups: Vec::new(), + successor_offsets: vec![0], + successors: Vec::new(), + predecessor_offsets: vec![0], + predecessors: Vec::new(), + row_offsets: Vec::new(), + uses: Vec::new(), + defs: Vec::new(), + phi_out: Vec::new(), + total_cells: 0, + }; + let mut selected = HashSet::new(); + let mut predecessors: Vec> = Vec::new(); + for &module_index in indices { + if !selected.insert(module_index) { + return Err("Duplicate subset function index".into()); + } + let function = module + .functions + .get(module_index) + .ok_or_else(|| format!("Invalid subset function index {module_index}"))?; + let blocks = function.blocks.len(); + if blocks == 0 || function.value_count >= MAX_VALUES { + return Err(format!("Invalid analysis dimensions in {}", function.name)); + } + let words = function.value_count.div_ceil(32).max(1); + let cells = blocks + .checked_mul(words) + .ok_or("Analysis dimensions overflow")?; + if cells > MAX_CELLS - packed.uses.len() { + return Err("Workload exceeds 100 million bitset words".into()); + } + let base = predecessors.len(); + let end = base + .checked_add(blocks) + .ok_or("Block dimensions overflow")?; + if end > u32::MAX as usize { + return Err("Too many CFG blocks".into()); + } + let row_base = packed.uses.len(); + predecessors.resize_with(end, Vec::new); + packed.functions.push(FunctionDesc { + module_index, + block_base: base, + block_count: blocks, + row_base, + words, + value_count: function.value_count, + }); + for word in 0..words { + packed.groups.push(GPUGroup { + block_base: base as u32, + block_count: blocks as u32, + row_base: row_base as u32, + words: words as u32, + word: word as u32, + }); + } + for (block_index, block) in function.blocks.iter().enumerate() { + packed.row_offsets.push(packed.uses.len()); + let mut local_uses = vec![0u32; words]; + let mut local_defs = vec![0u32; words]; + for instruction in &block.instructions { + if !instruction.is_phi { + for &value in &instruction.uses { + validate_value(value, function.value_count, &function.name)?; + let (word, mask) = bit(value); + // Only uses before a same-block definition contribute + // to block GEN. All phi operands live on CFG edges. + local_uses[word] |= mask & !local_defs[word]; + } + } + if let Some(value) = instruction.definition { + validate_value(value, function.value_count, &function.name)?; + let (word, mask) = bit(value); + local_defs[word] |= mask; + } + } + packed.uses.extend(local_uses); + packed.defs.extend(local_defs); + packed.phi_out.resize(packed.uses.len(), 0); + for &target in &block.successors { + if target >= blocks { + return Err(format!("Invalid CFG target in {}", function.name)); + } + if packed.successors.len() == u32::MAX as usize { + return Err("Too many CFG edges".into()); + } + packed.successors.push((base + target) as u32); + predecessors[base + target].push((base + block_index) as u32); + } + packed + .successor_offsets + .push(packed.successors.len() as u32); + } + for edge in &function.phi_edges { + if edge.from >= blocks + || edge.to >= blocks + || !function.blocks[edge.from].successors.contains(&edge.to) + { + return Err(format!("Invalid phi edge in {}", function.name)); + } + let row = row_base + edge.from * words; + for &value in &edge.values { + validate_value(value, function.value_count, &function.name)?; + let (word, mask) = bit(value); + packed.phi_out[row + word] |= mask; + } + } + } + for incoming in predecessors { + if incoming.len() > u32::MAX as usize - packed.predecessors.len() { + return Err("Too many predecessor edges".into()); + } + packed.predecessors.extend(incoming); + packed + .predecessor_offsets + .push(packed.predecessors.len() as u32); + } + packed.total_cells = packed.uses.len(); + packed.row_offsets.push(packed.total_cells); + Ok(packed) +} + +fn validate_value(value: usize, count: usize, name: &str) -> Result<(), String> { + if value < count { + Ok(()) + } else { + Err(format!("Invalid SSA value {value} in {name}")) + } +} + +fn bit(value: usize) -> (usize, u32) { + (value / 32, 1u32 << (value % 32)) +} + +/// Select functions from prepared structured IR. Copies and CFG remapping are +/// explicit recurring costs, and original ModuleIR function identities survive. +/// No frontend parsing or GEN/DEF reconstruction participates in this operation. +pub fn select_packed( + original: &PackedAnalysis, + module_indices: &[usize], +) -> Result { + let mut selected = HashSet::new(); + let mut result = PackedAnalysis { + functions: Vec::new(), + groups: Vec::new(), + successor_offsets: vec![0], + successors: Vec::new(), + predecessor_offsets: vec![0], + predecessors: Vec::new(), + row_offsets: Vec::new(), + uses: Vec::new(), + defs: Vec::new(), + phi_out: Vec::new(), + total_cells: 0, + }; + for &module_index in module_indices { + if !selected.insert(module_index) { + return Err("Duplicate subset function index".into()); + } + let source = find_function(original, module_index) + .ok_or("Subset function absent from original analysis")?; + let block_base = result.row_offsets.len(); + let row_base = result.uses.len(); + let cells = source.block_count * source.words; + result + .uses + .extend_from_slice(&original.uses[source.row_base..source.row_base + cells]); + result + .defs + .extend_from_slice(&original.defs[source.row_base..source.row_base + cells]); + result + .phi_out + .extend_from_slice(&original.phi_out[source.row_base..source.row_base + cells]); + result.functions.push(FunctionDesc { + module_index, + block_base, + block_count: source.block_count, + row_base, + words: source.words, + value_count: source.value_count, + }); + for word in 0..source.words { + result.groups.push(GPUGroup { + block_base: block_base as u32, + block_count: source.block_count as u32, + row_base: row_base as u32, + words: source.words as u32, + word: word as u32, + }); + } + for local in 0..source.block_count { + let block = source.block_base + local; + result.row_offsets.push(row_base + local * source.words); + for edge in original.successor_range(block) { + let target = original.successors[edge] as usize; + if target < source.block_base || target >= source.block_base + source.block_count { + return Err("Subset CFG edge escapes owning function".into()); + } + result + .successors + .push((block_base + target - source.block_base) as u32); + } + result + .successor_offsets + .push(result.successors.len() as u32); + for edge in original.predecessor_range(block) { + let predecessor = original.predecessors[edge] as usize; + if predecessor < source.block_base + || predecessor >= source.block_base + source.block_count + { + return Err("Subset predecessor escapes owning function".into()); + } + result + .predecessors + .push((block_base + predecessor - source.block_base) as u32); + } + result + .predecessor_offsets + .push(result.predecessors.len() as u32); + } + } + result.total_cells = result.uses.len(); + result.row_offsets.push(result.total_cells); + Ok(result) +} + +pub fn solve_cpu_serial(packed: &PackedAnalysis) -> Vec { + let mut result = vec![0; packed.total_cells]; + for function in &packed.functions { + let solution = solve_words(packed, function, 0..function.words); + let end = function.row_base + solution.len(); + result[function.row_base..end].copy_from_slice(&solution); + } + result +} + +/// Adaptively avoid starting worker threads for small compiler inputs. +pub fn solve_cpu(packed: &PackedAnalysis) -> Vec { + if packed.functions.len() < 16 && packed.total_cells < 4_096 { + solve_cpu_serial(packed) + } else { + solve_cpu_parallel(packed) + } +} + +/// Functions are independent. Workers share immutable CSR input and write +/// disjoint contiguous output slices; each worker solves many functions. +pub fn solve_cpu_parallel(packed: &PackedAnalysis) -> Vec { + let workers = worker_count(packed.functions.len()); + if workers <= 1 { + return solve_cpu_serial(packed); + } + let mut result = vec![0; packed.total_cells]; + let per_worker = packed.functions.len().div_ceil(workers); + std::thread::scope(|scope| { + let mut remainder = result.as_mut_slice(); + for functions in packed.functions.chunks(per_worker) { + let start = functions[0].row_base; + let last = functions.last().unwrap(); + let end = last.row_base + last.block_count * last.words; + let (destination, rest) = remainder.split_at_mut(end - start); + remainder = rest; + scope.spawn(move || { + for function in functions { + let solution = solve_words(packed, function, 0..function.words); + let local = function.row_base - start; + destination[local..local + solution.len()].copy_from_slice(&solution); + } + }); + } + }); + result +} + +/// Four-word chunks expose parallelism in a workload with only a few wide +/// functions. All scheduling, temporary output and scatter costs are included. +pub fn solve_cpu_word_parallel(packed: &PackedAnalysis) -> Vec { + let mut tasks = Vec::new(); + for (index, function) in packed.functions.iter().enumerate() { + for first in (0..function.words).step_by(4) { + tasks.push((index, first..(first + 4).min(function.words))); + } + } + let workers = worker_count(tasks.len()); + if workers <= 1 { + return solve_cpu_serial(packed); + } + let next = AtomicUsize::new(0); + let chunks = std::thread::scope(|scope| { + let handles: Vec<_> = (0..workers) + .map(|_| { + let tasks = &tasks; + let next = &next; + scope.spawn(move || { + let mut output = Vec::new(); + loop { + let task = next.fetch_add(1, Ordering::Relaxed); + if task >= tasks.len() { + break; + } + let (function, words) = &tasks[task]; + output.push(( + task, + solve_words(packed, &packed.functions[*function], words.clone()), + )); + } + output + }) + }) + .collect(); + handles + .into_iter() + .map(|h| h.join().expect("CPU liveness worker panicked")) + .collect::>() + }); + let mut result = vec![0; packed.total_cells]; + for worker in chunks { + for (task, solution) in worker { + let (function, words) = &tasks[task]; + let function = &packed.functions[*function]; + let width = words.len(); + for block in 0..function.block_count { + let destination = function.row_base + block * function.words + words.start; + result[destination..destination + width] + .copy_from_slice(&solution[block * width..(block + 1) * width]); + } + } + } + result +} + +fn worker_count(tasks: usize) -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .min(tasks) +} + +pub(crate) fn solve_words( + packed: &PackedAnalysis, + function: &FunctionDesc, + range: Range, +) -> Vec { + let blocks = function.block_count; + let words = range.len(); + let mut live = vec![0u32; blocks * words]; + let mut scratch = vec![0u32; words]; + // A circular queue holds each block at most once. Reverse block order + // traverses ordinary forward CFGs in one visit per block. + let mut queue: Vec = (0..blocks).rev().collect(); + let mut queued = vec![true; blocks]; + let (mut head, mut tail, mut pending) = (0, 0, blocks); + while pending > 0 { + let local = queue[head]; + head += 1; + if head == blocks { + head = 0; + } + pending -= 1; + queued[local] = false; + let block = function.block_base + local; + let row = function.row_base + local * function.words + range.start; + scratch.copy_from_slice(&packed.phi_out[row..row + words]); + for edge in packed.successor_range(block) { + let target = packed.successors[edge] as usize - function.block_base; + for (word, scratch_word) in scratch.iter_mut().enumerate() { + *scratch_word |= live[target * words + word]; + } + } + let mut changed = false; + for (word, scratch_word) in scratch.iter().enumerate() { + let value = packed.uses[row + word] | (*scratch_word & !packed.defs[row + word]); + let cell = local * words + word; + if live[cell] != value { + live[cell] = value; + changed = true; + } + } + if changed { + for edge in packed.predecessor_range(block) { + let predecessor = packed.predecessors[edge] as usize - function.block_base; + if !queued[predecessor] { + queue[tail] = predecessor; + tail += 1; + if tail == blocks { + tail = 0; + } + pending += 1; + queued[predecessor] = true; + } + } + } + } + live +} + +#[derive(Clone, Debug)] +pub struct HybridPlan { + pub cpu_indices: Vec, + pub gpu_indices: Vec, + pub candidate_gpu_functions: usize, + pub candidate_gpu_cells: usize, + pub candidate_gpu_word_groups: usize, + pub reason: &'static str, +} + +pub const MAX_GPU_BLOCKS: usize = 64; +pub const MIN_GPU_FUNCTIONS: usize = 32; +pub const MIN_GPU_CELLS: usize = 50_000; +pub const MIN_GPU_WORD_GROUPS: usize = 1_024; + +pub fn hybrid_plan(packed: &PackedAnalysis) -> HybridPlan { + let candidates: Vec<_> = packed + .functions + .iter() + .filter(|f| f.block_count <= MAX_GPU_BLOCKS) + .collect(); + let count = candidates.len(); + let cells = candidates.iter().map(|f| f.block_count * f.words).sum(); + let groups = candidates.iter().map(|f| f.words).sum(); + let admit = + count >= MIN_GPU_FUNCTIONS && (cells >= MIN_GPU_CELLS || groups >= MIN_GPU_WORD_GROUPS); + let mut cpu_indices = Vec::new(); + let mut gpu_indices = Vec::new(); + for function in &packed.functions { + if admit && function.block_count <= MAX_GPU_BLOCKS { + gpu_indices.push(function.module_index); + } else { + cpu_indices.push(function.module_index); + } + } + HybridPlan { + cpu_indices, + gpu_indices, + candidate_gpu_functions: count, + candidate_gpu_cells: cells, + candidate_gpu_word_groups: groups, + reason: if admit { + "GPU batch admission thresholds met" + } else { + "GPU batch below admission thresholds" + }, + } +} + +pub fn scatter( + subset: &PackedAnalysis, + source: &[u32], + original: &PackedAnalysis, + destination: &mut [u32], +) -> Result<(), String> { + if source.len() != subset.total_cells || destination.len() != original.total_cells { + return Err("Analysis scatter dimensions do not match".into()); + } + for function in &subset.functions { + let target = find_function(original, function.module_index) + .ok_or("Subset function absent from original analysis")?; + if target.block_count != function.block_count || target.words != function.words { + return Err("Analysis scatter function dimensions do not match".into()); + } + let count = function.block_count * function.words; + destination[target.row_base..target.row_base + count] + .copy_from_slice(&source[function.row_base..function.row_base + count]); + } + Ok(()) +} + +fn find_function(packed: &PackedAnalysis, module_index: usize) -> Option<&FunctionDesc> { + // Full module packing retains source order, so scattering and pruning + // normally resolve a function directly instead of repeatedly scanning. + packed + .functions + .get(module_index) + .filter(|f| f.module_index == module_index) + .or_else(|| { + packed + .functions + .iter() + .find(|f| f.module_index == module_index) + }) +} + +/// Remove dead pure scalar instructions, preserving every call, memory +/// operation, phi node, and control-flow operation. Phi incoming operands are +/// seeded in predecessor live-out; they must keep their definitions alive. +pub fn prune( + module: &ModuleIR, + packed: &PackedAnalysis, + solution: &[u32], +) -> Result<(String, usize), String> { + if solution.len() != packed.total_cells || packed.functions.len() != module.functions.len() { + return Err("Pruning requires a complete analysis solution".into()); + } + let mut output = String::with_capacity(module.text.len()); + output.push_str(&module.preamble); + let mut removed = 0; + for (module_index, function) in module.functions.iter().enumerate() { + let desc = find_function(packed, module_index) + .ok_or("Pruning analysis omitted a module function")?; + output.push_str(&function.header); + if !function.header.ends_with('\n') { + output.push('\n'); + } + for (local_block, block) in function.blocks.iter().enumerate() { + let global = desc.block_base + local_block; + let row = desc.row_base + local_block * desc.words; + let mut live = packed.phi_out[row..row + desc.words].to_vec(); + for edge in packed.successor_range(global) { + let target = packed.successors[edge] as usize; + let target_row = packed.row_offsets[target]; + for word in 0..desc.words { + live[word] |= solution[target_row + word]; + } + } + let mut keep = vec![true; block.instructions.len()]; + for (index, instruction) in block.instructions.iter().enumerate().rev() { + if let Some(value) = instruction.definition { + let (word, mask) = bit(value); + if instruction.pure && !instruction.is_phi && live[word] & mask == 0 { + keep[index] = false; + removed += 1; + continue; + } + live[word] &= !mask; + } + if !instruction.is_phi { + for &value in &instruction.uses { + let (word, mask) = bit(value); + live[word] |= mask; + } + } + } + output.push_str(&block.name); + output.push_str(":\n"); + for (instruction, keep) in block.instructions.iter().zip(keep) { + if keep { + output.push_str(&instruction.text); + if !instruction.text.ends_with('\n') { + output.push('\n'); + } + } + } + } + output.push_str(&function.footer); + if !function.footer.ends_with('\n') { + output.push('\n'); + } + } + output.push_str(&module.postamble); + Ok((output, removed)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::{BlockIR, FunctionIR, InstructionIR, PhiEdge}; + use std::collections::BTreeSet; + + fn instruction( + text: &str, + definition: Option, + uses: &[usize], + pure: bool, + phi: bool, + ) -> InstructionIR { + InstructionIR { + text: format!(" {text}"), + definition, + uses: uses.to_vec(), + pure, + is_phi: phi, + } + } + + fn block(name: &str, successors: &[usize], instructions: Vec) -> BlockIR { + BlockIR { + name: name.into(), + successors: successors.to_vec(), + instructions, + } + } + + fn module(functions: Vec) -> ModuleIR { + ModuleIR { + text: String::new(), + preamble: "; test\n".into(), + functions, + postamble: String::new(), + } + } + + fn function( + name: &str, + values: usize, + blocks: Vec, + phi_edges: Vec, + ) -> FunctionIR { + FunctionIR { + name: name.into(), + header: format!("define i64 @{name}() {{\n"), + footer: "}\n".into(), + value_count: values, + blocks, + phi_edges, + } + } + + /// Deliberately independent set equations, rebuilding GEN/DEF from + /// instructions and scanning the entire CFG rather than packed rows. + fn oracle(module: &ModuleIR, packed: &PackedAnalysis) -> Vec { + let mut result = vec![0; packed.total_cells]; + for desc in &packed.functions { + let function = &module.functions[desc.module_index]; + let mut gen = Vec::new(); + let mut defs = Vec::new(); + let mut phi = vec![BTreeSet::new(); function.blocks.len()]; + for block in &function.blocks { + let mut g = BTreeSet::new(); + let mut d = BTreeSet::new(); + for instruction in &block.instructions { + if !instruction.is_phi { + for value in &instruction.uses { + if !d.contains(value) { + g.insert(*value); + } + } + } + if let Some(value) = instruction.definition { + d.insert(value); + } + } + gen.push(g); + defs.push(d); + } + for edge in &function.phi_edges { + phi[edge.from].extend(edge.values.iter().copied()); + } + let mut live = vec![BTreeSet::new(); function.blocks.len()]; + loop { + let previous = live.clone(); + for (index, block) in function.blocks.iter().enumerate() { + let mut out = phi[index].clone(); + for &target in &block.successors { + out.extend(previous[target].iter().copied()); + } + live[index] = gen[index] + .union(&out.difference(&defs[index]).copied().collect()) + .copied() + .collect(); + } + if live == previous { + break; + } + } + for (block, values) in live.iter().enumerate() { + for &value in values { + let (word, mask) = bit(value); + result[desc.row_base + block * desc.words + word] |= mask; + } + } + } + result + } + + #[test] + fn phi_incoming_is_live_out_but_killed_by_predecessor_definition() { + let source = module(vec![function( + "phi", + 4, + vec![ + block( + "entry", + &[1], + vec![ + instruction("%a = add i64 7, 1", Some(0), &[], true, false), + instruction("%unused = add i64 5, 6", Some(3), &[], true, false), + instruction("br label %join", None, &[], false, false), + ], + ), + block( + "join", + &[], + vec![ + instruction("%p = phi i64 [ %a, %entry ]", Some(1), &[], false, true), + instruction("ret i64 %p", None, &[1], false, false), + ], + ), + ], + vec![PhiEdge { + from: 0, + to: 1, + values: vec![0], + }], + )]); + let packed = prepare(&source).unwrap(); + let actual = solve_cpu_serial(&packed); + assert_eq!(actual, oracle(&source, &packed)); + assert_eq!(actual, vec![0, 0]); + assert_eq!(packed.phi_out, vec![1, 0]); + let (output, removed) = prune(&source, &packed, &actual).unwrap(); + assert!(output.contains("%a = add")); + assert!(output.contains("%p = phi")); + assert!(!output.contains("%unused")); + assert_eq!(removed, 1); + } + + #[test] + fn loops_word_boundaries_and_parallel_variants_match_set_oracle() { + let mut functions = Vec::new(); + for index in 0..37 { + functions.push(function( + &format!("loop{index}"), + 161, + vec![ + block( + "entry", + &[1], + vec![instruction( + "br label %loop", + None, + &[0, 31, 32, 63, 64, 127, 160], + false, + false, + )], + ), + block( + "loop", + &[1, 2], + vec![ + instruction("%x = add i64 %arg, 1", Some(1), &[0], true, false), + instruction( + "br i1 %condition, label %loop, label %exit", + None, + &[32], + false, + false, + ), + ], + ), + block( + "exit", + &[], + vec![instruction("ret i64 %x", None, &[1, 160], false, false)], + ), + ], + vec![], + )); + } + let source = module(functions); + let packed = prepare(&source).unwrap(); + let expected = oracle(&source, &packed); + assert_eq!(solve_cpu_serial(&packed), expected); + assert_eq!(solve_cpu(&packed), expected); + assert_eq!(solve_cpu_parallel(&packed), expected); + assert_eq!(solve_cpu_word_parallel(&packed), expected); + assert_eq!(std::mem::size_of::(), 20); + } + + #[test] + fn pruning_preserves_effects_and_removes_dead_dependency_chain() { + let source = module(vec![function( + "effects", + 8, + vec![block( + "entry", + &[], + vec![ + instruction("%a = add i64 1, 2", Some(0), &[], true, false), + instruction("%b = mul i64 %a, 3", Some(1), &[0], true, false), + instruction("%ptr = alloca i64", Some(2), &[], false, false), + instruction("%load = load i64, ptr %ptr", Some(3), &[2], false, false), + instruction( + "%call = call i64 @observe(i64 %load)", + Some(4), + &[3], + false, + false, + ), + instruction("store i64 %call, ptr %ptr", None, &[4, 2], false, false), + instruction("call void @llvm.trap()", None, &[], false, false), + instruction("ret i64 0", None, &[], false, false), + ], + )], + vec![], + )]); + let packed = prepare(&source).unwrap(); + let (output, removed) = prune(&source, &packed, &solve_cpu(&packed)).unwrap(); + assert_eq!(removed, 2); + for retained in [ + "alloca", + "load i64", + "@observe", + "store i64", + "@llvm.trap", + "ret i64", + ] { + assert!(output.contains(retained)); + } + } + + #[test] + fn hybrid_admission_scatter_and_empty_subset() { + let mut functions = Vec::new(); + for index in 0..32 { + functions.push(function( + &format!("wide{index}"), + 1024, + vec![block( + "entry", + &[], + vec![instruction("ret i64 %arg", None, &[index], false, false)], + )], + vec![], + )); + } + functions.push(function( + "long", + 1, + (0..65) + .map(|block_index| { + let successors = if block_index == 64 { + Vec::new() + } else { + vec![block_index + 1] + }; + block( + &format!("b{block_index}"), + &successors, + vec![instruction("ret i64 %arg", None, &[0], false, false)], + ) + }) + .collect(), + vec![], + )); + let source = module(functions); + let packed = prepare(&source).unwrap(); + let plan = hybrid_plan(&packed); + assert_eq!(plan.candidate_gpu_word_groups, 1024); + assert_eq!(plan.gpu_indices.len(), 32); + assert_eq!(plan.cpu_indices, vec![32]); + let mut merged = vec![0; packed.total_cells]; + for indices in [&plan.cpu_indices, &plan.gpu_indices] { + let subset = prepare_subset(&source, indices).unwrap(); + scatter(&subset, &solve_cpu_serial(&subset), &packed, &mut merged).unwrap(); + } + assert_eq!(merged, solve_cpu_serial(&packed)); + let empty = prepare_subset(&source, &[]).unwrap(); + assert_eq!(solve_cpu(&empty), Vec::::new()); + assert_eq!(empty.row_offsets, vec![0]); + let small = prepare_subset(&source, &[0]).unwrap(); + assert!(hybrid_plan(&small).gpu_indices.is_empty()); + } + + #[test] + fn invalid_phi_edges_and_values_fail_closed() { + let mut source = module(vec![function( + "bad", + 1, + vec![block("entry", &[], vec![])], + vec![PhiEdge { + from: 0, + to: 0, + values: vec![0], + }], + )]); + assert!(prepare(&source).unwrap_err().contains("Invalid phi edge")); + source.functions[0].phi_edges.clear(); + source.functions[0].blocks[0].instructions.push(instruction( + "bad", + None, + &[1], + false, + false, + )); + assert!(prepare(&source).unwrap_err().contains("Invalid SSA value")); + assert!(prepare_subset(&source, &[2]).is_err()); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-bitcode-linker.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-bitcode-linker.rs new file mode 100644 index 00000000000..b5ebf32ce50 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-bitcode-linker.rs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gpu_rust_compiler::bitcode_linker::{run_with, ProcessRunner, WrapperOptions}; +use std::path::PathBuf; + +fn main() { + let result = (|| { + let mut args = std::env::args_os().skip(1); + let rustc = PathBuf::from( + args.next() + .ok_or("RUSTC_WRAPPER requires the original rustc path")?, + ); + run_with( + &rustc, + &args.collect::>(), + &WrapperOptions::from_env()?, + &mut ProcessRunner, + ) + })(); + match result { + Ok(outcome) => { + if let Some(error) = outcome.report.error { + eprintln!("GPU object bridge: {error}"); + } + std::process::exit(outcome.exit_code); + } + Err(error) => { + eprintln!("GPU object bridge: {error}"); + std::process::exit(1); + } + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-cargo.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-cargo.rs new file mode 100644 index 00000000000..ecfb86c9efb --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-cargo.rs @@ -0,0 +1,271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gpu_rust_compiler::cargo_policy::{ + self, ObjectBackend, Options, Placement, Quality, SizeBasis, +}; +use std::{ + env, fs, + io::Write, + path::{Path, PathBuf}, + time::Instant, +}; + +const HELP: &str = "Usage: gpu-cargo plan|build --manifest-path Cargo.toml --package PACKAGE --bin BINARY [OPTIONS] + --quality fast|balanced|release Default: fast + --placement auto|cpu|metal|cuda Required GPUs need the GPU object adapter + --object-backend native|tpde|llvm-bitcode|gpu Default: native + --bitcode-wrapper FILE Explicit two-phase rustc wrapper + --tpde-controller FILE Object-emission controller + --tpde-bridge FILE Pinned LLVM/TPDE shared bridge + --llvm-llc FILE LLVM object emitter + --allow-backend-fallback Permit attributed CPU object fallback + --gpu-backend metal|cuda Device API for GPU object adapter + --gpu-emitter FILE Qualified module-to-GPU emitter + --gpu-library FILE Native GPU code-emission library + --gpu-shader FILE Required Metal code-emission shader + --gpu-device INDEX Default: physical device index 0 + --target-root DIRECTORY Owned quality directories below DIRECTORY + --offline Build only from cached dependencies + --features FEATURES Cargo feature selection + --no-default-features Cargo feature selection + --reference-artifact EXECUTABLE Declared generated-size baseline + --max-size-ratio RATIO Default: 1.5; requires a reference when specified + --size-basis unmodified|strip-debug Same complete-file basis for candidate and reference + --report FILE Save the same JSON emitted to stdout + --dry-run Plan without building or creating target directories + +Uses pinned rustc 1.98.1. Alternate object adapters support fast/balanced; +release preserves the native Cargo LTO contract. Actual GPU work requires +physical module-emission receipts, not a requested device label."; + +fn value(arguments: &mut impl Iterator, name: &str) -> Result { + arguments + .next() + .ok_or_else(|| format!("Missing value for {name}")) +} + +fn parse() -> Result)>, String> { + let mut arguments = env::args().skip(1); + let action = arguments.next().ok_or(HELP)?; + if matches!(action.as_str(), "--help" | "-h") { + println!("{HELP}"); + return Ok(None); + } + let plan = match action.as_str() { + "plan" => true, + "build" => false, + _ => return Err(HELP.into()), + }; + let mut options = Options { + plan, + manifest_path: PathBuf::from("Cargo.toml"), + package: String::new(), + binary: String::new(), + quality: Quality::Fast, + placement: Placement::Auto, + target_root: None, + offline: false, + features: None, + no_default_features: false, + reference_artifact: None, + max_size_ratio: 1.5, + size_basis: SizeBasis::Unmodified, + object_backend: ObjectBackend::Native, + bitcode_wrapper: None, + tpde_controller: None, + tpde_bridge: None, + llvm_llc: None, + allow_backend_fallback: false, + gpu_emitter: None, + gpu_library: None, + gpu_shader: None, + gpu_backend: None, + gpu_device: 0, + }; + let mut report = None; + let mut ratio_supplied = false; + while let Some(argument) = arguments.next() { + match argument.as_str() { + "--help" | "-h" => { + println!("{HELP}"); + return Ok(None); + } + "--manifest-path" => options.manifest_path = value(&mut arguments, &argument)?.into(), + "--package" | "-p" => options.package = value(&mut arguments, &argument)?, + "--bin" => options.binary = value(&mut arguments, &argument)?, + "--quality" => options.quality = Quality::parse(&value(&mut arguments, &argument)?)?, + "--placement" => { + options.placement = Placement::parse(&value(&mut arguments, &argument)?)? + } + "--object-backend" => { + options.object_backend = ObjectBackend::parse(&value(&mut arguments, &argument)?)? + } + "--bitcode-wrapper" => { + options.bitcode_wrapper = Some(value(&mut arguments, &argument)?.into()) + } + "--tpde-controller" => { + options.tpde_controller = Some(value(&mut arguments, &argument)?.into()) + } + "--tpde-bridge" => options.tpde_bridge = Some(value(&mut arguments, &argument)?.into()), + "--llvm-llc" => options.llvm_llc = Some(value(&mut arguments, &argument)?.into()), + "--allow-backend-fallback" => options.allow_backend_fallback = true, + "--gpu-emitter" => options.gpu_emitter = Some(value(&mut arguments, &argument)?.into()), + "--gpu-library" => options.gpu_library = Some(value(&mut arguments, &argument)?.into()), + "--gpu-shader" => options.gpu_shader = Some(value(&mut arguments, &argument)?.into()), + "--gpu-backend" => { + options.gpu_backend = Some(Placement::parse(&value(&mut arguments, &argument)?)?) + } + "--gpu-device" => { + options.gpu_device = value(&mut arguments, &argument)? + .parse() + .map_err(|_| "Invalid GPU device index")? + } + "--target-root" => options.target_root = Some(value(&mut arguments, &argument)?.into()), + "--offline" => options.offline = true, + "--features" => options.features = Some(value(&mut arguments, &argument)?), + "--no-default-features" => options.no_default_features = true, + "--reference-artifact" => { + options.reference_artifact = Some(value(&mut arguments, &argument)?.into()) + } + "--max-size-ratio" => { + ratio_supplied = true; + options.max_size_ratio = value(&mut arguments, &argument)? + .parse() + .map_err(|_| "Invalid size ratio")?; + } + "--size-basis" => { + options.size_basis = SizeBasis::parse(&value(&mut arguments, &argument)?)? + } + "--report" => report = Some(PathBuf::from(value(&mut arguments, &argument)?)), + "--dry-run" => options.plan = true, + _ => return Err(format!("Unknown option {argument}")), + } + } + if ratio_supplied && options.reference_artifact.is_none() { + return Err("--max-size-ratio requires --reference-artifact".into()); + } + if let Some(path) = &report { + validate_report_path(path)?; + for protected in [ + &options.manifest_path, + &options.manifest_path.with_file_name("Cargo.lock"), + ] { + if same_path(path, protected) { + return Err("Report path would overwrite a Cargo input".into()); + } + } + if options + .reference_artifact + .as_ref() + .is_some_and(|reference| same_path(path, reference)) + { + return Err("Report path would overwrite the reference artifact".into()); + } + } + Ok(Some((options, report))) +} + +fn same_path(left: &Path, right: &Path) -> bool { + let normalize = |path: &Path| { + fs::canonicalize(path).unwrap_or_else(|_| { + if path.is_absolute() { + path.to_owned() + } else { + env::current_dir().unwrap_or_default().join(path) + } + }) + }; + normalize(left) == normalize(right) +} + +fn write_report(path: &Path, text: &str) -> Result<(), String> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + let temporary = path.with_extension(format!("gpu-cargo-{}.tmp", std::process::id())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|e| e.to_string())?; + let result = file + .write_all(format!("{text}\n").as_bytes()) + .map_err(|e| e.to_string()); + drop(file); + let result = result.and_then(|_| fs::rename(&temporary, path).map_err(|e| e.to_string())); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn validate_report_path(path: &Path) -> Result<(), String> { + if path.exists() { + let existing = fs::read(path).map_err(|e| e.to_string())?; + let owned = serde_json::from_slice::(&existing) + .is_ok_and(|value| value["schema"] == "gpu-cargo-report" && value["version"] == 1); + if !owned { + return Err( + "Refusing to replace an existing file that is not a gpu-cargo report".into(), + ); + } + } + Ok(()) +} + +fn run() -> Result { + let Some((options, path)) = parse()? else { + return Ok(true); + }; + let start = Instant::now(); + let report = match cargo_policy::execute(&options) { + Ok(report) => report, + Err(error) => { + let text = serde_json::to_string(&serde_json::json!({ + "schema":"gpu-cargo-report", "version":1, "status":"error", "error":error, + "quality":options.quality, "requested_placement":options.placement, + "object_backend":options.object_backend, + "toolchain":cargo_policy::TOOLCHAIN, "actual_backend":null, + "gpu_compilation_jobs":0, "manifest_path":options.manifest_path, + "total_elapsed_ms":start.elapsed().as_secs_f64()*1000.0, + })) + .map_err(|e| e.to_string())?; + if let Some(path) = path { + write_report(&path, &text)?; + } + println!("{text}"); + eprintln!("gpu-cargo: {error}"); + return Ok(false); + } + }; + if let (Some(path), Some(artifact)) = (&path, &report.artifact) { + if same_path(path, &artifact.path) { + return Err("Report path would overwrite the compiled artifact".into()); + } + } + let text = serde_json::to_string(&report).map_err(|e| e.to_string())?; + if let Some(path) = path { + write_report(&path, &text)?; + } + println!("{text}"); + Ok(matches!(report.status, "ok" | "planned")) +} + +fn main() { + match run() { + Ok(true) => {} + Ok(false) => std::process::exit(1), + Err(error) => { + println!( + "{}", + serde_json::json!({"schema":"gpu-cargo-report", "version":1, "status":"error", "error":error}) + ); + eprintln!("gpu-cargo: {error}"); + std::process::exit(1); + } + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-codegen.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-codegen.rs new file mode 100644 index 00000000000..c47d84005de --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-codegen.rs @@ -0,0 +1,391 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use gpu_rust_compiler::{ + codegen_service::*, + machine_codegen::{self, MachineTarget}, + native_codegen::NativeExecutor, +}; +use sha2::{Digest, Sha256}; +use std::{ + collections::VecDeque, + env, fs, + path::{Path, PathBuf}, + process::Command, + sync::Arc, + time::Instant, +}; + +fn hash(data: &[u8]) -> String { + Sha256::digest(data) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +fn constants(index: usize, seed: u64, generation: u64) -> (i64, i64) { + let a = seed + .wrapping_add(index as u64 * 7919) + .wrapping_add(generation * 104729) + .rotate_left(17); + (a as i64, (a ^ 0x8000000013572468) as i64) +} +fn batch( + start: usize, + count: usize, + seed: u64, + generation: u64, + target: MachineTarget, + source: &str, +) -> Result, String> { + let mut functions = Vec::new(); + let mut instructions = Vec::new(); + let mut symbols = Vec::new(); + for index in start..start + count { + let symbol = functions.len() as u32; + let begin = instructions.len() as u32; + let (a, b) = constants(index, seed, generation); + let values = [ + (Opcode::Argument, Some(0), [0, 0], 0), + (Opcode::Const, Some(1), [0, 0], a), + (Opcode::Add, Some(2), [0, 1], 0), + (Opcode::Const, Some(3), [0, 0], b), + (Opcode::Sub, Some(4), [2, 3], 0), + (Opcode::Mul, Some(5), [4, 1], 0), + (Opcode::Copy, Some(6), [5, 0], 0), + (Opcode::Return, None, [6, 0], 0), + ]; + instructions.extend( + values + .into_iter() + .map(|(opcode, result, operands, immediate)| Instruction { + opcode, + result, + operands, + immediate, + }), + ); + functions.push(Function { + symbol, + instruction_start: begin, + instruction_count: 8, + value_count: 7, + abi: 0, + }); + symbols.push(Symbol { + name: format!("gpu_emitted_{index}"), + }); + } + Ok(Arc::new(FlatBatch::new( + BatchIdentity { + workload_id: format!("synthetic-leaf:{start}:{count}:{seed}:{generation}"), + toolchain_id: "rust1.98.1-leaf-schema1".into(), + policy_id: "fast-leaf-stack-v1".into(), + target_abi: target.host_triple().into(), + source_revision: source.into(), + }, + BatchKey { + logical_id: format!("leaf-{start}"), + generation, + }, + functions, + instructions, + symbols, + vec![Abi { + calling_convention: if target == MachineTarget::Aarch64 && cfg!(target_os = "macos") { + CallingConvention::AppleAarch64 + } else { + CallingConvention::SystemV + }, + parameter_count: 1, + returns_i64: true, + requires_unwind: false, + }], + )?)) +} +fn execute( + objects: &[PathBuf], + count: usize, + seed: u64, + generation: u64, + out: &Path, +) -> Result { + let mut c = String::from("#include \n#include \n#include \n"); + for i in 0..count { + c.push_str(&format!("extern uint64_t gpu_emitted_{i}(uint64_t);\n")); + } + c.push_str("int main(void){uint64_t xs[]={0,1,UINT64_MAX,INT64_MAX,(uint64_t)INT64_MIN,UINT64_C(0x123456789abcdef0)};uint64_t h=UINT64_C(14695981039346656037);\n"); + for i in 0..count { + c.push_str(&format!( + "for(unsigned j=0;j<6;j++){{h^=gpu_emitted_{i}(xs[j]);h*=UINT64_C(1099511628211);}}\n" + )); + } + c.push_str("printf(\"%\" PRIu64 \"\\n\",h);return 0;}\n"); + fs::write(out.join("main.c"), c).map_err(|e| e.to_string())?; + let link = Instant::now(); + let mut cmd = Command::new("clang"); + cmd.arg(out.join("main.c")) + .args(objects) + .arg("-o") + .arg(out.join("program")); + let result = cmd.output().map_err(|e| e.to_string())?; + let link_ms = link.elapsed().as_secs_f64() * 1000.; + if !result.status.success() { + return Err(format!( + "native fixture link failed: {}", + String::from_utf8_lossy(&result.stderr) + )); + } + let run = Command::new(out.join("program")) + .output() + .map_err(|e| e.to_string())?; + if !run.status.success() { + return Err("emitted native fixture failed to execute".into()); + } + let mut expected = 14695981039346656037u64; + for i in 0..count { + let (a, b) = constants(i, seed, generation); + for x in [0i64, 1, -1, i64::MAX, i64::MIN, 0x123456789abcdef0] { + let value = x.wrapping_add(a).wrapping_sub(b).wrapping_mul(a); + expected ^= value as u64; + expected = expected.wrapping_mul(1099511628211); + } + } + let expected = format!("{expected}\n"); + let actual = String::from_utf8(run.stdout).map_err(|e| e.to_string())?; + if actual != expected { + return Err(format!( + "native program mismatch: expected{expected:?}, actual{actual:?}" + )); + } + Ok( + serde_json::json!({"verified":true,"stdout":actual,"link_ms":link_ms,"program_sha256":hash(&fs::read(out.join("program")).map_err(|e|e.to_string())?)}), + ) +} +fn run() -> Result<(), String> { + let mut backend = String::from("cpu"); + let mut library = None; + let mut shader = None; + let mut functions = 128usize; + let mut batch_size = 256usize; + let mut repeats = 3usize; + let mut workers = 1usize; + let mut devices = vec![0u32]; + let mut seed = 73u64; + let mut out = PathBuf::from(".build/codegen-demo"); + let mut execute_program = false; + let mut args = env::args().skip(1); + while let Some(arg) = args.next() { + if arg == "--help" { + println!("gpu-codegen --backend cpu|metal|cuda --functions N --batch-functions N --repeats N --cpu-workers N --devices 0,1 --library FILE --shader FILE --seed N --out-dir DIR --execute\nSchema-1 leaf machine-code fixture; no full Rust/Cargo GPU support or scaling claim."); + return Ok(()); + } + if arg == "--execute" { + execute_program = true; + continue; + } + let value = args + .next() + .ok_or_else(|| format!("missing value for{arg}"))?; + match arg.as_str() { + "--backend" => backend = value, + "--library" => library = Some(PathBuf::from(value)), + "--shader" => shader = Some(PathBuf::from(value)), + "--functions" => functions = value.parse().map_err(|_| "invalid functions")?, + "--batch-functions" => batch_size = value.parse().map_err(|_| "invalid batch size")?, + "--repeats" => repeats = value.parse().map_err(|_| "invalid repeats")?, + "--cpu-workers" => workers = value.parse().map_err(|_| "invalid workers")?, + "--devices" => { + devices = value + .split(',') + .map(|v| v.parse().map_err(|_| "invalid device index")) + .collect::>()? + } + "--seed" => seed = value.parse().map_err(|_| "invalid seed")?, + "--out-dir" => out = value.into(), + _ => return Err(format!("unknown option{arg}")), + } + } + if !(1..=65536).contains(&functions) + || !(1..=4096).contains(&batch_size) + || !(1..=64).contains(&repeats) + || !(1..=64).contains(&workers) + || devices.is_empty() + { + return Err("fixture dimensions or worker counts are out of bounds".into()); + } + let mut unique = devices.clone(); + unique.sort(); + unique.dedup(); + if unique.len() != devices.len() { + return Err("duplicate device index".into()); + } + let marker = out.join(".gpuemit-owned"); + if out.exists() && !marker.exists() { + return Err("refusing unowned output directory".into()); + } + fs::create_dir_all(&out).map_err(|e| e.to_string())?; + fs::write(&marker, "gpuemit-schema1\n").map_err(|e| e.to_string())?; + let target = MachineTarget::host()?; + let git_source = Command::new("git") + .args(["rev-parse", "HEAD"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()); + let ci_source = env::var("CANDIDATE_SHA") + .ok() + .filter(|sha| matches!(sha.len(), 40 | 64) && sha.bytes().all(|c| c.is_ascii_hexdigit())); + let (source, source_origin) = if let Some(sha) = git_source { + (sha, "git-checkout") + } else if let Some(sha) = ci_source { + (sha, "ci-environment-assertion") + } else { + ("unrecorded-source".into(), "unknown") + }; + let dirty = Command::new("git") + .args(["status", "--porcelain"]) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| !o.stdout.is_empty()); + let initialization = Instant::now(); + let mut executors: Vec> = Vec::new(); + let mut stats = Vec::new(); + let placement = if backend == "cpu" { + for worker in 0..workers { + executors.push(Box::new(CpuCallbackExecutor::new( + format!("cpu-{worker}"), + move |batch| machine_codegen::cpu_emit(batch, target), + ))); + } + Placement::Cpu + } else { + let kind = match backend.as_str() { + "metal" => GpuBackend::Metal, + "cuda" => GpuBackend::Cuda, + _ => return Err("backend must be cpu, metal, or cuda".into()), + }; + let library = library + .as_ref() + .ok_or("GPU emission requires explicit library path")?; + for device in &devices { + let executor = NativeExecutor::new(kind, target, library, shader.as_deref(), *device)?; + stats.push(executor.stats()); + executors.push(Box::new(executor)); + } + Placement::RequireGpu(kind) + }; + let session = BuildSession::new( + SessionConfig { + cpu_credit_limit: workers.max(devices.len()), + max_queued_batches: 32, + ..Default::default() + }, + executors, + ) + .map_err(|e| e.to_string())?; + let client = session + .register_client("native-codegen", workers.max(devices.len())) + .map_err(|e| e.to_string())?; + let setup_ms = initialization.elapsed().as_secs_f64() * 1000.; + let mut rounds = Vec::new(); + for round in 0..repeats { + let generation = round as u64 + 1; + let start = Instant::now(); + let mut pending = VecDeque::new(); + let mut completed = Vec::new(); + for first in (0..functions).step_by(batch_size) { + let input = batch( + first, + batch_size.min(functions - first), + seed, + generation, + target, + &source, + )?; + loop { + match session.submit( + &client, + input.clone(), + SubmitOptions { + placement: placement.clone(), + estimated_work: input.instructions().len() as u64, + ..Default::default() + }, + ) { + Ok(ticket) => { + pending.push_back((input.clone(), ticket)); + break; + } + Err(ServiceError::Backpressure) => { + let (input, ticket) = pending + .pop_front() + .ok_or("backpressure without pending work")?; + completed.push((input, ticket.wait().map_err(|e| e.to_string())?)); + } + Err(e) => return Err(e.to_string()), + } + } + } + for (input, ticket) in pending { + completed.push((input, ticket.wait().map_err(|e| e.to_string())?)); + } + let emit_wall_ms = start.elapsed().as_secs_f64() * 1000.; + let object_start = Instant::now(); + let mut objects = Vec::new(); + let mut ledgers = Vec::new(); + let mut code_bytes = 0usize; + for (index, (input, receipt)) in completed.iter().enumerate() { + if !session.is_current(receipt) { + return Err("stale completion cannot be published".into()); + } + let cpu = machine_codegen::cpu_emit(input, target)?; + let mut oracle = cpu.functions; + oracle.sort_by_key(|f| f.symbol); + let mut actual = receipt.output.functions.clone(); + actual.sort_by_key(|f| f.symbol); + if oracle != actual { + return Err("GPU/native emission differs from independent CPU bytes".into()); + } + code_bytes += actual.iter().map(|f| f.code.len()).sum::(); + let bytes = machine_codegen::write_object(input, target, &receipt.output)?; + let path = out.join(format!("round{round}-batch{index}.o")); + fs::write(&path, bytes).map_err(|e| e.to_string())?; + objects.push(path); + let c = &receipt.costs; + ledgers.push(serde_json::json!({"device":receipt.route.device_id,"physical_gpu":receipt.execution.is_physical_gpu(),"validation_ms":c.validation.as_secs_f64()*1000.,"queue_ms":c.queue.as_secs_f64()*1000.,"prepare_ms":c.prepare.as_secs_f64()*1000.,"submit_ms":c.submit.as_secs_f64()*1000.,"wait_ms":c.wait.as_secs_f64()*1000.,"output_validation_ms":c.assemble.as_secs_f64()*1000.})); + } + let oracle_and_objects_ms = object_start.elapsed().as_secs_f64() * 1000.; + let execution = if execute_program { + execute(&objects, functions, seed, generation, &out)? + } else { + serde_json::json!({"verified":false,"reason":"native execution not requested"}) + }; + rounds.push(serde_json::json!({"round":round,"generation":generation,"emission_wall_ms":emit_wall_ms,"oracle_and_objects_ms":oracle_and_objects_ms,"code_bytes":code_bytes,"ledger":ledgers,"program_execution":execution})); + } + let native_stats: Vec<_> = stats.iter().map(|s| s.lock().unwrap().clone()).collect(); + let exe = env::current_exe().map_err(|e| e.to_string())?; + let library_hash = library + .as_ref() + .map(|p| fs::read(p).map(|v| hash(&v))) + .transpose() + .map_err(|e| e.to_string())?; + let shader_hash = shader + .as_ref() + .map(|p| fs::read(p).map(|v| hash(&v))) + .transpose() + .map_err(|e| e.to_string())?; + let report = serde_json::json!({"schema_version":1,"scope":"Synthetic schema-1 leaf machine-code emission and native execution; not full Rust/Cargo GPU compilation or demonstrated hardware scaling.","backend":backend,"target":target.host_triple(),"functions":functions,"batch_functions":batch_size,"cpu_workers":workers,"requested_devices":devices,"device_ids":session.capabilities().iter().map(|d|d.device_id.clone()).collect::>(),"service_initialization_ms":setup_ms,"seed":seed,"quality":"fast-leaf-stack-v1","all_cpu_byte_checks_passed":true,"native_stats":native_stats,"rounds":rounds,"source_revision":source,"source_revision_origin":source_origin,"worktree_dirty":dirty,"binary_sha256":hash(&fs::read(exe).map_err(|e|e.to_string())?),"native_library_sha256":library_hash,"shader_sha256":shader_hash,"fixture_spec_sha256":hash(include_bytes!("gpu-codegen.rs")),"comparison_scope":"Emission wall includes fixture construction/validation, queue, preparation, native submission/wait and output validation; independent oracle, object serialization/linking/execution are separately reported."}); + fs::write( + out.join("report.json"), + serde_json::to_vec_pretty(&report).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + println!("{}", out.join("report.json").display()); + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("gpu-codegen: {error}"); + std::process::exit(1); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-corpus-census.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-corpus-census.rs new file mode 100644 index 00000000000..19689f0d11c --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-corpus-census.rs @@ -0,0 +1,480 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gpu_rust_compiler::{leaf_packet, machine_codegen}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet}, + env, fs, + io::Write, + path::{Component, Path, PathBuf}, + process::Command, + time::Instant, +}; + +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +fn export_outcome(code: i32, diagnostic: &str, packet: Option<&[u8]>) -> Result { + match code { + 0 if packet.is_some_and(|p| p.len() > 8 && p.starts_with(b"GEM1")) => Ok(true), + 0 => Err("successful export did not produce a GEM1 packet".into()), + 20 if !diagnostic.trim().is_empty() && packet.is_none() => Ok(false), + _ => Err(format!("leaf inspection failed ({code}): {diagnostic}")), + } +} + +fn capture_path(directory: &Path, name: &str) -> Result { + let path = Path::new(name); + if path.components().count() != 1 + || !matches!(path.components().next(), Some(Component::Normal(_))) + || path.extension().and_then(|s| s.to_str()) != Some("bc") + { + return Err("capture filename must be a bitcode basename".into()); + } + let candidate = directory.join(path); + if fs::symlink_metadata(&candidate) + .map_err(|e| e.to_string())? + .file_type() + .is_symlink() + { + return Err("captured input cannot be a symlink".into()); + } + fs::canonicalize(candidate).map_err(|e| e.to_string()) +} + +fn verify_input(bytes: &[u8], sha: &str, size: u64) -> Result<(), String> { + if (!bytes.starts_with(b"BC\xc0\xde") && !bytes.starts_with(b"\xde\xc0\x17\x0b")) + || bytes.len() as u64 != size + || hash(bytes) != sha + { + return Err("capture bytes, length or hash differ from the manifest".into()); + } + Ok(()) +} + +fn matched_inputs(actual: &[String], expected: &[String]) -> Result<(), String> { + fn counts(inputs: &[String]) -> BTreeMap<&str, usize> { + let mut counts = BTreeMap::new(); + for input in inputs { + *counts.entry(input.as_str()).or_insert(0usize) += 1; + } + counts + } + if counts(actual) != counts(expected) { + return Err("captured input hashes differ from the completed wrapper receipts".into()); + } + Ok(()) +} + +fn bind_capture(capture: &Value, receipt: &Value) -> Result<(), String> { + for key in [ + "status", + "source", + "input_arguments_sha256", + "quality", + "requested_backend", + "rustc_version_verbose", + ] { + if capture.get(key).is_none() || capture.get(key) != receipt.get(key) { + return Err(format!( + "capture header differs from wrapper receipt: {key}" + )); + } + } + let captured = capture + .get("units") + .and_then(Value::as_array) + .ok_or("capture has no units")?; + let emitted = receipt + .get("units") + .and_then(Value::as_array) + .ok_or("receipt has no units")?; + if captured.len() != emitted.len() { + return Err("capture unit count differs from its wrapper receipt".into()); + } + for (index, (capture_unit, emitted_unit)) in captured.iter().zip(emitted).enumerate() { + if integer(capture_unit, "unit_index")? != index as u64 + || capture_unit.get("emission_receipt") != Some(emitted_unit) + || capture_unit.get("target") != emitted_unit.get("target") + || capture_unit.get("controller_elapsed_ms") + != emitted_unit.get("controller_elapsed_ms") + || capture_unit.get("input_sha256") != emitted_unit.get("input_sha256") + || capture_unit.get("llvm_codegen_opt_level") != capture.get("llvm_codegen_opt_level") + || capture_unit.get("llvm_codegen_opt_level") + != emitted_unit.pointer("/controller_receipt/llvm_codegen_opt_level") + { + return Err( + "captured unit identity, codegen level or cost differs from its emission receipt" + .into(), + ); + } + } + Ok(()) +} + +fn read_json(path: &Path) -> Result { + serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) + .map_err(|e| format!("{}: {e}", path.display())) +} +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str, String> { + value + .get(key) + .and_then(Value::as_str) + .ok_or_else(|| format!("missing string {key}")) +} +fn integer(value: &Value, key: &str) -> Result { + value + .get(key) + .and_then(Value::as_u64) + .ok_or_else(|| format!("missing integer {key}")) +} + +fn run() -> Result<(), String> { + let mut options = BTreeMap::new(); + let mut args = env::args().skip(1); + while let Some(key) = args.next() { + if key == "--help" { + println!("gpu-corpus-census --corpus-root DIR --build-report JSON --bridge FILE --packet-dir NEW_DIR --report NEW_JSON\nInspect every captured Rust LLVM module with the strict whole-module GPU exporter. This audit does not measure GPU acceleration."); + return Ok(()); + } + if ![ + "--corpus-root", + "--build-report", + "--bridge", + "--packet-dir", + "--report", + ] + .contains(&key.as_str()) + { + return Err(format!("unknown option {key}")); + } + let value = args + .next() + .ok_or_else(|| format!("missing value for {key}"))?; + if options.insert(key.clone(), value).is_some() { + return Err(format!("duplicate {key}")); + } + } + let required = |key: &str| -> Result { + options + .get(key) + .map(PathBuf::from) + .ok_or_else(|| format!("missing {key}")) + }; + let root = fs::canonicalize(required("--corpus-root")?).map_err(|e| e.to_string())?; + let build_path = fs::canonicalize(required("--build-report")?).map_err(|e| e.to_string())?; + let build = read_json(&build_path)?; + if text(&build, "schema")? != "gpu-cargo-report" + || text(&build, "status")? != "ok" + || text(&build, "object_backend")? != "llvm-bitcode" + || build + .pointer("/artifact/cargo_fresh") + .and_then(Value::as_bool) + != Some(false) + { + return Err("a successful fresh LLVM-bitcode Cargo audit is required".into()); + } + let receipts = build + .get("object_backend_receipts") + .ok_or("missing wrapper receipts")?; + let expected = integer(receipts, "translated_units")?; + if expected == 0 || integer(receipts, "failed_semantic_invocations")? != 0 { + return Err( + "capture build contains no translated units or a failed semantic invocation".into(), + ); + } + let mut expected_inputs = vec![]; + let mut raw_captures = BTreeMap::new(); + for entry in fs::read_dir(text(receipts, "directory")?).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if !entry.file_type().map_err(|e| e.to_string())?.is_file() { + return Err("wrapper receipt directory contains a non-file".into()); + } + let receipt = read_json(&entry.path())?; + if let Some(capture_path) = receipt.get("capture_manifest").and_then(Value::as_str) { + let capture_path = fs::canonicalize(capture_path).map_err(|e| e.to_string())?; + if !capture_path.starts_with(&root) + || raw_captures.insert(capture_path, receipt.clone()).is_some() + { + return Err( + "wrapper capture path is outside the selected corpus or duplicated".into(), + ); + } + } + for unit in receipt + .get("units") + .and_then(Value::as_array) + .ok_or("wrapper receipt has no units")? + { + expected_inputs.push(text(unit, "input_sha256")?.to_string()); + } + } + if expected_inputs.len() as u64 != expected { + return Err("raw wrapper receipts differ from the aggregate module count".into()); + } + let bridge = fs::canonicalize(required("--bridge")?).map_err(|e| e.to_string())?; + if build + .pointer("/backend_toolchain/bridge/sha256") + .and_then(Value::as_str) + != Some(hash(&fs::read(&bridge).map_err(|e| e.to_string())?).as_str()) + { + return Err("census bridge differs from capture build provenance".into()); + } + let packet_dir = required("--packet-dir")?; + fs::create_dir(&packet_dir).map_err(|e| format!("claim new packet directory: {e}"))?; + let packet_dir = fs::canonicalize(packet_dir).map_err(|e| e.to_string())?; + let report_path = required("--report")?; + if report_path.exists() { + return Err("report already exists".into()); + } + let mut manifests = vec![]; + for entry in fs::read_dir(&root).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if !entry.file_type().map_err(|e| e.to_string())?.is_dir() { + return Err("capture root must contain only owned invocation directories".into()); + } + manifests.push(entry.path().join("manifest.json")); + } + manifests.sort(); + if manifests.len() != raw_captures.len() { + return Err("capture manifests differ from the completed invocation receipts".into()); + } + let time = Instant::now(); + let mut units = vec![]; + let mut seen_paths = BTreeSet::new(); + let mut input_hashes = BTreeSet::new(); + let mut captured_hashes = vec![]; + let mut reasons = BTreeMap::::new(); + let mut eligible_functions = 0; + let mut eligible_emission_sum_ms = 0.; + let mut eligibility_sum_ms = 0.; + let mut corpus_hash = Sha256::new(); + for path in &manifests { + let capture = read_json(path)?; + let canonical = fs::canonicalize(path).map_err(|e| e.to_string())?; + bind_capture( + &capture, + raw_captures + .get(&canonical) + .ok_or("capture is absent from completed invocation receipts")?, + )?; + if text(&capture, "schema")? != "gpu-rust-bitcode-capture" + || integer(&capture, "version")? != 1 + || text(&capture, "status")? != "success" + || text(&capture, "quality")? != text(&build, "quality")? + { + return Err(format!( + "capture manifest is incomplete or incompatible: {}", + path.display() + )); + } + let captured = capture + .get("units") + .and_then(Value::as_array) + .ok_or("missing captured units")?; + for unit in captured { + let input = capture_path(path.parent().unwrap(), text(unit, "captured_filename")?)?; + if !seen_paths.insert(input.clone()) { + return Err("captured input appears more than once".into()); + } + let bytes = fs::read(&input).map_err(|e| e.to_string())?; + let input_sha = text(unit, "input_sha256")?; + verify_input(&bytes, input_sha, integer(unit, "input_bytes")?)?; + input_hashes.insert(input_sha.to_string()); + captured_hashes.push(input_sha.to_string()); + corpus_hash.update(input_sha.as_bytes()); + let packet_path = packet_dir.join(format!("module-{}.gem1", units.len())); + let inspection = Instant::now(); + let export = Command::new(&bridge) + .arg("--export-leaf") + .arg(&input) + .arg(&packet_path) + .output() + .map_err(|e| format!("start leaf exporter: {e}"))?; + let packet = if packet_path.exists() { + Some(fs::read(&packet_path).map_err(|e| e.to_string())?) + } else { + None + }; + let mut diagnostic = String::from_utf8_lossy(&export.stderr).trim().to_string(); + let mut eligible = export_outcome( + export.status.code().unwrap_or(-1), + &diagnostic, + packet.as_deref(), + )?; + let mut functions = 0; + if eligible { + let decoded = leaf_packet::decode(packet.as_deref().unwrap())?; + let emission = machine_codegen::cpu_emit(&decoded.batch, decoded.target)?; + match machine_codegen::write_object(&decoded.batch, decoded.target, &emission) { + Ok(_) => functions = decoded.batch.functions().len(), + Err(error) => { + eligible = false; + diagnostic = format!("GPU object preflight rejected: {error}"); + } + } + } + let eligibility_ms = inspection.elapsed().as_secs_f64() * 1000.; + eligibility_sum_ms += eligibility_ms; + let mut row = json!({ + "input_path":input, "input_sha256":input_sha, "input_bytes":bytes.len(), + "capture_manifest":path, "source":capture.get("source"), + "llvm_codegen_opt_level":unit.get("llvm_codegen_opt_level"), + "target":unit.get("target"), "eligibility_ms":eligibility_ms, + "cpu_controller_elapsed_ms":unit.get("controller_elapsed_ms"), + "status":if eligible {"eligible"} else {"rejected"} + }); + if eligible { + let packet = packet.unwrap(); + eligible_functions += functions; + eligible_emission_sum_ms += unit + .get("controller_elapsed_ms") + .and_then(Value::as_f64) + .ok_or("missing captured emission timing")?; + row["packet_path"] = json!(packet_path); + row["packet_sha256"] = json!(hash(&packet)); + row["functions"] = json!(functions); + } else { + // The full diagnostic stays on the row; aggregate stable exporter + // reasons independently of mangled function names. + let reason = diagnostic + .strip_prefix("GPU leaf export rejected: ") + .unwrap_or(&diagnostic); + let reason = reason + .rsplit_once(": ") + .map_or(reason, |(_, tail)| tail) + .to_string(); + *reasons.entry(reason).or_default() += 1; + row["reason"] = json!(diagnostic); + } + units.push(row); + } + } + if units.len() as u64 != expected { + return Err(format!( + "incomplete corpus: captured {} of {expected} translated units", + units.len() + )); + } + matched_inputs(&captured_hashes, &expected_inputs)?; + let eligible = units.iter().filter(|u| u["status"] == "eligible").count(); + let report = json!({ + "schema":"gpu-rust-corpus-census", "version":1, + "corpus_id":corpus_hash.finalize().iter().map(|b| format!("{b:02x}")).collect::(), + "source_revision":build.pointer("/source/git_head").and_then(Value::as_str).unwrap_or("unrecorded"), + "source_worktree_dirty":build.pointer("/source/git_worktree_dirty"), + "toolchain_id":"rust1.98.1-llvm22.1.8", + "build_report":build_path, "build_report_sha256":hash(&fs::read(&build_path).map_err(|e| e.to_string())?), + "bridge":bridge, "bridge_sha256":build.pointer("/backend_toolchain/bridge/sha256"), + "scope":"Strict whole-module export and object-contract eligibility census of every captured fast-build LLVM unit; no GPU timing or whole-build speedup claim", + "exhaustive_capture_verified":true, "total_units":units.len(), + "unique_input_hashes":input_hashes.len(), "eligible_units":eligible, + "eligible_functions":eligible_functions, "rejected_units":units.len()-eligible, + "capture_invocations":manifests.len(), "rejection_reasons":reasons, + "inspection_wall_ms":time.elapsed().as_secs_f64()*1000., + "eligibility_sum_ms":eligibility_sum_ms, + "eligible_cpu_controller_elapsed_sum_ms":eligible_emission_sum_ms, + "timing_scope":"Controller sums overlap and are not whole-build wall-time shares", + "build_cargo_elapsed_ms":build.get("cargo_elapsed_ms"), + "continuation_gate":{"required_whole_fast_build_improvement":0.10, + "compiler_size_limit":1.5,"program_size_limit":1.5, + "whole_build_speedup_demonstrated":false, + "decision":if eligible==0 {"park-current-gpu-route-zero-eligible-work"} else {"replay-required-no-performance-qualification"}, + "zero_cost_gpu_available_work_ms":if eligible==0 {Some(0.)} else {None}}, + "units":units, + }); + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&report_path) + .map_err(|e| e.to_string())?; + file.write_all(&serde_json::to_vec_pretty(&report).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + file.sync_all().map_err(|e| e.to_string())?; + println!( + "Inspected {} Rust modules: {eligible} eligible, {} rejected. {}", + units.len(), + units.len() - eligible, + report["continuation_gate"]["decision"] + ); + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("corpus census: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_leaf_is_rejected_but_broken_compiler_is_an_error() { + assert!(!export_outcome( + 20, + "GPU leaf export rejected: globals are unsupported", + None + ) + .unwrap()); + assert!(export_outcome(21, "LLVM parse failed", None).is_err()); + } + + #[test] + fn success_without_a_packet_cannot_establish_eligibility() { + assert!(export_outcome(0, "", None).is_err()); + assert!(export_outcome(0, "", Some(b"bad packet")).is_err()); + assert!(export_outcome(20, "", None).is_err()); + } + + #[test] + fn changed_capture_bytes_cannot_qualify() { + let bytes = b"BC\xc0\xdeactual"; + assert!(verify_input(bytes, &hash(bytes), bytes.len() as u64).is_ok()); + assert!(verify_input(bytes, &hash(b"different"), bytes.len() as u64).is_err()); + assert!(verify_input(bytes, &hash(bytes), 2).is_err()); + } + + #[test] + fn matching_counts_with_missing_modules_cannot_establish_exhaustiveness() { + assert!(matched_inputs(&["a".into(), "a".into()], &["a".into(), "b".into()]).is_err()); + assert!(matched_inputs(&["b".into(), "a".into()], &["a".into(), "b".into()]).is_ok()); + } + + #[test] + fn capture_costs_are_bound_to_the_exact_emitted_unit() { + let mut receipt = json!({ + "status":"success", "source":"/source.rs", "input_arguments_sha256":"arg", + "quality":"fast", "requested_backend":"llvm", "rustc_version_verbose":"pinned", + "units":[{"input_sha256":"input", "target":"x86_64-unknown-linux-gnu", + "controller_elapsed_ms":2.5, "controller_receipt":{"llvm_codegen_opt_level":1}}] + }); + let mut capture = receipt.clone(); + capture["llvm_codegen_opt_level"] = json!(1); + capture["units"] = json!([{ + "unit_index":0,"input_sha256":"input", "target":"x86_64-unknown-linux-gnu", + "controller_elapsed_ms":2.5,"llvm_codegen_opt_level":1,"emission_receipt":receipt["units"][0] + }]); + assert!(bind_capture(&capture, &receipt).is_ok()); + capture["units"][0]["controller_elapsed_ms"] = json!(9.); + assert!(bind_capture(&capture, &receipt).is_err()); + capture["units"][0]["controller_elapsed_ms"] = json!(2.5); + receipt["source"] = json!("/another.rs"); + assert!(bind_capture(&capture, &receipt).is_err()); + } + + #[test] + fn capture_names_cannot_escape_the_invocation_directory() { + for name in ["../unit.bc", "/unit.bc", "nested/unit.bc", "unit.o"] { + assert!(capture_path(Path::new("/unused"), name).is_err()); + } + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-corpus-replay.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-corpus-replay.rs new file mode 100644 index 00000000000..c1d7493a2b0 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-corpus-replay.rs @@ -0,0 +1,570 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use gpu_rust_compiler::{ + codegen_service::*, + leaf_packet, + machine_codegen::{self, MachineTarget}, + native_codegen::NativeExecutor, +}; +use object::Object as _; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + env, fs, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::Arc, + time::Instant, +}; + +#[derive(Deserialize)] +struct Manifest { + schema: String, + version: u32, + corpus_id: String, + source_revision: String, + toolchain_id: String, + total_units: usize, + eligible_units: usize, + rejected_units: usize, + rejection_reasons: BTreeMap, + exhaustive_capture_verified: bool, + #[serde(default)] + inspection_wall_ms: f64, + units: Vec, +} +#[derive(Deserialize)] +struct Unit { + input_path: PathBuf, + input_sha256: String, + status: String, + packet_path: Option, + packet_sha256: Option, + target: Option, + eligibility_ms: f64, + reason: Option, +} +struct CapturedModule { + input_sha256: String, + packet_sha256: String, + packet: Vec, + target: MachineTarget, + target_abi: String, + functions: usize, + instructions: usize, +} +struct Corpus { + manifest: Manifest, + modules: Vec, + duplicate_eligible_units: usize, + eligibility_sum_ms: f64, +} +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +fn file_hash(path: &Path) -> Result { + if !path.is_absolute() { + return Err("census input and packet paths must be absolute".into()); + } + let mut file = fs::File::open(path).map_err(|e| format!("read {}: {e}", path.display()))?; + let mut digest = Sha256::new(); + let mut buffer = [0u8; 64 * 1024]; + loop { + let count = file.read(&mut buffer).map_err(|e| e.to_string())?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + Ok(digest + .finalize() + .iter() + .map(|b| format!("{b:02x}")) + .collect()) +} +fn load(path: &Path) -> Result { + let manifest: Manifest = serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if manifest.schema != "gpu-rust-corpus-census" + || manifest.version != 1 + || !manifest.exhaustive_capture_verified + || manifest.toolchain_id != "rust1.98.1-llvm22.1.8" + || manifest.source_revision.is_empty() + || manifest.corpus_id.is_empty() + || manifest.total_units != manifest.units.len() + || manifest.eligible_units.checked_add(manifest.rejected_units) + != Some(manifest.total_units) + || manifest.rejection_reasons.values().sum::() != manifest.rejected_units + || !manifest.inspection_wall_ms.is_finite() + || manifest.inspection_wall_ms < 0. + { + return Err("unverified, inconsistent or incompatible corpus census".into()); + } + let mut paths = BTreeSet::new(); + let mut hashes = BTreeMap::)>::new(); + let mut modules = Vec::new(); + let mut eligible = 0; + let mut rejected = 0; + let mut duplicate_eligible_units = 0; + let mut eligibility_sum_ms = 0.; + let mut packet_bytes = 0usize; + for unit in &manifest.units { + if !unit.eligibility_ms.is_finite() || unit.eligibility_ms < 0. { + return Err("invalid eligibility timing".into()); + } + eligibility_sum_ms += unit.eligibility_ms; + if !paths.insert(fs::canonicalize(&unit.input_path).map_err(|e| e.to_string())?) { + return Err("captured input path appears more than once".into()); + } + if file_hash(&unit.input_path)? != unit.input_sha256 { + return Err("captured input SHA256 differs from census".into()); + } + let duplicate = if let Some(previous) = hashes.get(&unit.input_sha256) { + if previous != &(unit.status.clone(), unit.packet_sha256.clone()) { + return Err("duplicate input hash has conflicting eligibility or packet".into()); + } + true + } else { + hashes.insert( + unit.input_sha256.clone(), + (unit.status.clone(), unit.packet_sha256.clone()), + ); + false + }; + match unit.status.as_str() { + "eligible" => { + eligible += 1; + let path = unit + .packet_path + .as_ref() + .ok_or("eligible module lacks packet path")?; + let expected = unit + .packet_sha256 + .as_ref() + .ok_or("eligible module lacks packet hash")?; + if file_hash(path)? != *expected { + return Err("eligible packet SHA256 differs from census".into()); + } + if duplicate { + duplicate_eligible_units += 1; + continue; + } + let length = fs::metadata(path).map_err(|e| e.to_string())?.len(); + if length > 128 * 1024 * 1024 { + return Err("eligible packet exceeds 128 MiB".into()); + } + packet_bytes = packet_bytes + .checked_add(length as usize) + .ok_or("packet size overflow")?; + if packet_bytes > 128 * 1024 * 1024 { + return Err("unique replay corpus exceeds 128 MiB packet memory limit".into()); + } + let packet = fs::read(path).map_err(|e| e.to_string())?; + // Recheck the owned snapshot, rather than trusting a prior file read. + if hash(&packet) != *expected { + return Err("eligible packet changed during snapshot".into()); + } + let decoded = leaf_packet::decode(&packet)?; + if let Some(target) = &unit.target { + if target != &decoded.llvm_target + && target != &decoded.batch.identity().target_abi + { + return Err("packet target differs from captured module target".into()); + } + } + let target_abi = decoded.batch.identity().target_abi.clone(); + if modules + .first() + .is_some_and(|m: &CapturedModule| m.target_abi != target_abi) + { + return Err("one replay corpus must use one target ABI".into()); + } + modules.push(CapturedModule { + input_sha256: unit.input_sha256.clone(), + packet_sha256: expected.clone(), + packet, + target: decoded.target, + target_abi, + functions: decoded.batch.functions().len(), + instructions: decoded.batch.instructions().len(), + }); + } + "rejected" => { + rejected += 1; + if unit.reason.as_deref().is_none_or(|s| s.is_empty()) + || unit.packet_path.is_some() + || unit.packet_sha256.is_some() + { + return Err( + "rejected module lacks reason or improperly carries an eligible packet" + .into(), + ); + } + } + _ => return Err("unknown census eligibility status".into()), + } + } + if eligible != manifest.eligible_units || rejected != manifest.rejected_units { + return Err("census eligibility totals differ from unit rows".into()); + } + Ok(Corpus { + manifest, + modules, + duplicate_eligible_units, + eligibility_sum_ms, + }) +} + +struct Side { + session: BuildSession, + client: Client, + placement: Placement, + initialization_ms: f64, +} +fn make_side( + executors: Vec>, + credits: usize, + placement: Placement, + initialization: Instant, +) -> Result { + let session = BuildSession::new( + SessionConfig { + cpu_credit_limit: credits, + ..Default::default() + }, + executors, + ) + .map_err(|e| e.to_string())?; + let client = session + .register_client("captured-module-replay", credits) + .map_err(|e| e.to_string())?; + Ok(Side { + session, + client, + placement, + initialization_ms: initialization.elapsed().as_secs_f64() * 1000., + }) +} +fn run_round( + side: &Side, + corpus: &Corpus, + round: usize, + expected_objects: &mut BTreeMap, +) -> Result { + let started = Instant::now(); + let mut pending = VecDeque::<(usize, Arc, Ticket)>::new(); + let mut completed = Vec::new(); + let mut decode_pack_ms = 0.; + for (index, module) in corpus.modules.iter().enumerate() { + let preparation = Instant::now(); + let decoded = leaf_packet::decode_with_key( + &module.packet, + BatchKey { + logical_id: module.input_sha256.clone(), + generation: round as u64 + 1, + }, + )?; + decode_pack_ms += preparation.elapsed().as_secs_f64() * 1000.; + loop { + match side.session.submit( + &side.client, + decoded.batch.clone(), + SubmitOptions { + placement: side.placement.clone(), + estimated_work: module.instructions as u64, + ..Default::default() + }, + ) { + Ok(ticket) => { + pending.push_back((index, decoded.batch.clone(), ticket)); + break; + } + Err(ServiceError::Backpressure) => { + let (index, batch, ticket) = pending.pop_front().ok_or( + "batch exceeds service memory budget; no pending work can release credits", + )?; + completed.push((index, batch, ticket.wait().map_err(|e| e.to_string())?)); + } + Err(error) => return Err(error.to_string()), + } + } + } + for (index, batch, ticket) in pending { + completed.push((index, batch, ticket.wait().map_err(|e| e.to_string())?)); + } + let emission_wall_ms = started.elapsed().as_secs_f64() * 1000.; + completed.sort_by_key(|(index, _, _)| *index); + let mut oracle_ms = 0.; + let mut object_serialization_ms = 0.; + let mut units = Vec::new(); + for (index, batch, receipt) in completed { + if !side.session.is_current(&receipt) { + return Err("stale replay output cannot be measured or published".into()); + } + let module = &corpus.modules[index]; + let oracle_start = Instant::now(); + let mut expected = machine_codegen::cpu_emit(&batch, module.target)?.functions; + expected.sort_by_key(|f| f.symbol); + let mut actual = receipt.output.functions.clone(); + actual.sort_by_key(|f| f.symbol); + if actual != expected { + return Err("captured module output differs from independent CPU byte oracle".into()); + } + oracle_ms += oracle_start.elapsed().as_secs_f64() * 1000.; + let object_start = Instant::now(); + let bytes = machine_codegen::write_object(&batch, module.target, &receipt.output)?; + if object::File::parse(bytes.as_slice()) + .map_err(|e| e.to_string())? + .kind() + != object::ObjectKind::Relocatable + { + return Err("replay emitted a nonrelocatable object".into()); + } + let object_sha = hash(&bytes); + if let Some(previous) = expected_objects.get(&module.input_sha256) { + if previous != &object_sha { + return Err("CPU/GPU or repeated corpus output is unstable".into()); + } + } else { + expected_objects.insert(module.input_sha256.clone(), object_sha.clone()); + } + object_serialization_ms += object_start.elapsed().as_secs_f64() * 1000.; + let (physical_gpu, kernel_dispatches, kernel_output_used) = match &receipt.execution { + ExecutionRecord::Cpu => (false, 0, false), + ExecutionRecord::PhysicalGpu { + kernel_dispatches, + kernel_output_used, + .. + } => (true, *kernel_dispatches, *kernel_output_used), + ExecutionRecord::Simulated { .. } => { + return Err("simulation cannot enter captured-corpus hardware comparison".into()) + } + }; + if matches!(side.placement, Placement::RequireGpu(_)) + && (!physical_gpu || kernel_dispatches == 0 || !kernel_output_used) + { + return Err("required GPU replay lacks physical kernel-output evidence".into()); + } + let costs = &receipt.costs; + units.push(json!({"input_sha256":module.input_sha256,"packet_sha256":module.packet_sha256,"functions":module.functions,"instructions":module.instructions,"packet_bytes":module.packet.len(),"object_bytes":bytes.len(),"object_sha256":object_sha,"device_id":receipt.route.device_id,"physical_gpu":physical_gpu,"kernel_dispatches":kernel_dispatches,"kernel_output_used":kernel_output_used,"worker_stages_ms":{"validation":costs.validation.as_secs_f64()*1000.,"queue":costs.queue.as_secs_f64()*1000.,"prepare":costs.prepare.as_secs_f64()*1000.,"submit":costs.submit.as_secs_f64()*1000.,"wait":costs.wait.as_secs_f64()*1000.,"output_validation":costs.assemble.as_secs_f64()*1000.}})); + } + let wall = started.elapsed().as_secs_f64() * 1000.; + Ok( + json!({"round":round,"unique_modules":corpus.modules.len(),"decode_validate_pack_sum_ms":decode_pack_ms,"emission_wall_ms":emission_wall_ms,"cpu_byte_oracle_ms":oracle_ms,"object_serialization_validation_hash_ms":object_serialization_ms,"total_wall_ms":wall,"units":units,"timing_scope":"total wall includes packet decode/validation/pack, concurrent service execution, wait, independent CPU byte oracle, unwind/object serialization, object validation and hashes; worker sums overlap and are not wall time"}), + ) +} +fn publish(path: &Path, report: &Value) -> Result<(), String> { + let bytes = serde_json::to_vec_pretty(report).map_err(|e| e.to_string())?; + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(path) + .map_err(|e| e.to_string())?; + if let Err(error) = file.write_all(&bytes).and_then(|_| file.sync_all()) { + drop(file); + let _ = fs::remove_file(path); + return Err(error.to_string()); + } + Ok(()) +} +fn median(values: &mut [f64]) -> f64 { + values.sort_by(f64::total_cmp); + let middle = values.len() / 2; + if values.len().is_multiple_of(2) { + (values[middle - 1] + values[middle]) / 2. + } else { + values[middle] + } +} +fn run() -> Result<(), String> { + let all = Instant::now(); + let mut options = BTreeMap::new(); + let mut arguments = env::args().skip(1); + while let Some(key) = arguments.next() { + if key == "--help" { + println!("gpu-corpus-replay --manifest CENSUS --report JSON --backend metal|cuda --library FILE [--shader FILE] [--devices 0,1] [--cpu-workers N] [--repeats N]\nReplay strict eligible whole Rust modules with retained CPU/GPU services; zero eligible modules stop before GPU initialization. Each unique module runs once per side per round; repetitions are not additional distinct work."); + return Ok(()); + } + if ![ + "--manifest", + "--report", + "--backend", + "--library", + "--shader", + "--devices", + "--cpu-workers", + "--repeats", + ] + .contains(&key.as_str()) + { + return Err(format!("unknown option {key}")); + } + let value = arguments + .next() + .ok_or_else(|| format!("missing value for {key}"))?; + if options.insert(key.clone(), value).is_some() { + return Err(format!("duplicate option {key}")); + } + } + let required = |key: &str| { + options + .get(key) + .cloned() + .ok_or_else(|| format!("missing {key}")) + }; + let manifest_path = PathBuf::from(required("--manifest")?); + let report_path = PathBuf::from(required("--report")?); + if report_path.exists() { + return Err("replay report must be a new file".into()); + } + let preparation = Instant::now(); + let corpus = load(&manifest_path)?; + let census_validation_ms = preparation.elapsed().as_secs_f64() * 1000.; + let mut report = json!({"schema":"gpu-rust-corpus-replay","version":1,"corpus_id":corpus.manifest.corpus_id,"source_revision":corpus.manifest.source_revision,"toolchain_id":corpus.manifest.toolchain_id,"manifest_sha256":hash(&fs::read(&manifest_path).map_err(|e|e.to_string())?),"total_units":corpus.manifest.total_units,"eligible_units":corpus.manifest.eligible_units,"rejected_units":corpus.manifest.rejected_units,"rejection_reasons":corpus.manifest.rejection_reasons,"exhaustive_capture_verified":corpus.manifest.exhaustive_capture_verified,"unique_replayed_units":corpus.modules.len(),"duplicate_eligible_units":corpus.duplicate_eligible_units,"census_validation_input_hash_snapshot_ms":census_validation_ms,"prior_census_inspection_wall_ms":corpus.manifest.inspection_wall_ms,"prior_census_eligibility_sum_ms":corpus.eligibility_sum_ms,"gpu_executed":false,"performance_speedup_demonstrated":false,"whole_build_speedup_demonstrated":false,"multi_gpu_scaling_demonstrated":false,"native_execution_verified":false,"scope":"captured whole Rust LLVM module code emission for strict eligible modules; excludes frontend, unsupported modules, CPU fallback, linker and generated-program runtime"}); + if corpus.modules.is_empty() { + report["decision"] = "stop-zero-eligible-modules".into(); + report["total_elapsed_ms"] = json!(all.elapsed().as_secs_f64() * 1000.); + publish(&report_path, &report)?; + println!("{}", report_path.display()); + return Ok(()); + } + let workers = options.get("--cpu-workers").map_or(Ok(4usize), |s| { + s.parse::().map_err(|_| "invalid CPU worker count") + })?; + let repeats = options.get("--repeats").map_or(Ok(5usize), |s| { + s.parse::().map_err(|_| "invalid repeat count") + })?; + let devices = options.get("--devices").map_or(Ok(vec![0]), |s| { + s.split(',') + .map(|s| s.parse::().map_err(|_| "invalid device index")) + .collect::, _>>() + })?; + if !(1..=64).contains(&workers) + || !(1..=64).contains(&repeats) + || devices.is_empty() + || devices.len() > 64 + || devices.iter().collect::>().len() != devices.len() + { + return Err("worker/repeat/device counts out of bounds or duplicate devices".into()); + } + let backend_name = required("--backend")?; + let backend = match backend_name.as_str() { + "metal" => GpuBackend::Metal, + "cuda" => GpuBackend::Cuda, + _ => return Err("backend must be metal or cuda".into()), + }; + let library = PathBuf::from(required("--library")?); + let shader = options.get("--shader").map(PathBuf::from); + let target = corpus.modules[0].target; + let cpu_started = Instant::now(); + let cpu_executors = (0..workers) + .map(|index| { + Box::new(CpuCallbackExecutor::new( + format!("cpu-{index}"), + move |batch| machine_codegen::cpu_emit(batch, target), + )) as Box + }) + .collect(); + let cpu = make_side(cpu_executors, workers, Placement::Cpu, cpu_started)?; + let gpu_started = Instant::now(); + let mut gpu_executors: Vec> = Vec::new(); + let mut native_stats = Vec::new(); + for device in &devices { + let executor = NativeExecutor::new(backend, target, &library, shader.as_deref(), *device)?; + native_stats.push(executor.stats()); + gpu_executors.push(Box::new(executor)); + } + let gpu = make_side( + gpu_executors, + devices.len(), + Placement::RequireGpu(backend), + gpu_started, + )?; + let mut expected_objects = BTreeMap::new(); + let mut rounds = Vec::new(); + let mut cpu_times = Vec::new(); + let mut gpu_times = Vec::new(); + for round in 0..repeats { + let (cpu_result, gpu_result) = if round.is_multiple_of(2) { + ( + run_round(&cpu, &corpus, round, &mut expected_objects)?, + run_round(&gpu, &corpus, round, &mut expected_objects)?, + ) + } else { + let gpu_result = run_round(&gpu, &corpus, round, &mut expected_objects)?; + ( + run_round(&cpu, &corpus, round, &mut expected_objects)?, + gpu_result, + ) + }; + cpu_times.push(cpu_result["total_wall_ms"].as_f64().unwrap()); + gpu_times.push(gpu_result["total_wall_ms"].as_f64().unwrap()); + rounds.push(json!({"round":round,"execution_order":if round%2==0 {"cpu-then-gpu"}else{"gpu-then-cpu"},"cpu":cpu_result,"gpu":gpu_result})); + } + let cpu_median = median(&mut cpu_times); + let gpu_median = median(&mut gpu_times); + let native_stats = native_stats + .iter() + .map(|s| s.lock().unwrap().clone()) + .collect::>(); + report["decision"] = "measured-eligible-corpus-only-no-whole-build-qualification".into(); + report["gpu_executed"] = true.into(); + report["backend"] = backend_name.into(); + report["cpu_workers"] = workers.into(); + report["requested_devices"] = json!(devices); + report["physical_device_ids"] = json!(gpu + .session + .capabilities() + .iter() + .map(|c| c.device_id.clone()) + .collect::>()); + report["persistent_sessions"] = true.into(); + report["scoped_cpu_credits"] = json!({"cpu_service":workers,"gpu_service":devices.len(),"cargo_jobserver_integration":false}); + report["repeats"] = repeats.into(); + report["distinct_work_amplification"] = false.into(); + report["cpu_service_initialization_ms"] = json!(cpu.initialization_ms); + report["gpu_service_initialization_ms"] = json!(gpu.initialization_ms); + report["native_stats_per_device"] = json!(native_stats); + report["all_cpu_byte_checks_passed"] = true.into(); + report["all_objects_stable"] = true.into(); + report["rounds"] = json!(rounds); + report["cpu_median_total_wall_ms"] = json!(cpu_median); + report["gpu_median_total_wall_ms"] = json!(gpu_median); + report["eligible_corpus_total_wall_cpu_over_gpu_ratio"] = json!(cpu_median / gpu_median); + report["amortized_prior_eligibility_export_sum_ms_per_round"] = + json!(corpus.eligibility_sum_ms / repeats as f64); + report["cpu_median_plus_amortized_startup_ms"] = + json!(cpu_median + cpu.initialization_ms / repeats as f64); + report["gpu_median_plus_amortized_startup_and_census_wall_ms"] = json!( + gpu_median + + (gpu.initialization_ms + census_validation_ms + corpus.manifest.inspection_wall_ms) + / repeats as f64 + ); + report["native_library_sha256"] = json!(file_hash( + &fs::canonicalize(&library).map_err(|e| e.to_string())? + )?); + report["shader_sha256"] = json!(shader + .as_ref() + .map(|s| fs::canonicalize(s) + .map_err(|e| e.to_string()) + .and_then(|s| file_hash(&s))) + .transpose()?); + report["binary_sha256"] = json!(file_hash(&env::current_exe().map_err(|e| e.to_string())?)?); + report["total_elapsed_ms"] = json!(all.elapsed().as_secs_f64() * 1000.); + report["timing_scope"] = "Complete corpus rounds alternate CPU/GPU order and retain both services; each unique captured module executes once per side per round. Round walls include decode/pack, service execution, oracle and object serialization. Census validation, prior exporter timing and retained-session startup are separate; worker-stage and exporter sums are not summed as wall time. No warmup or failed units excluded; no native object files are retained.".into(); + report["cpu_baseline_scope"] = "matched leaf machine-code emission algorithm using parallel CPU callbacks; not native LLVM/TPDE or a strongest full-Rust CPU baseline; CPU startup-adjusted wall excludes GPU-only eligibility/export/census cost".into(); + publish(&report_path, &report)?; + println!("{}", report_path.display()); + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("gpu-corpus-replay: {error}"); + std::process::exit(1); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-cpu-liveness.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-cpu-liveness.rs new file mode 100644 index 00000000000..df4dbfbf541 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-cpu-liveness.rs @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[allow(dead_code)] +// Benchmark the existing native solver unchanged, including its indexed +// bitset loop. The shared compiler module predates this benchmark executable. +#[allow(clippy::needless_range_loop)] +#[path = "../analysis.rs"] +mod analysis; +#[path = "../glc.rs"] +mod glc; +#[allow(dead_code)] +#[path = "../ir.rs"] +mod ir; + +use analysis::{solve_cpu_parallel, solve_cpu_serial, solve_cpu_word_parallel, PackedAnalysis}; +use std::{env, fs, io::Write, time::Instant}; + +struct Variant { + name: &'static str, + threads: usize, + first_ms: f64, + samples_ms: Vec, +} + +fn statistics(samples: &[f64]) -> String { + let mut sorted = samples.to_vec(); + sorted.sort_unstable_by(f64::total_cmp); + let middle = sorted.len() / 2; + let median = if sorted.len().is_multiple_of(2) { + (sorted[middle - 1] + sorted[middle]) / 2.0 + } else { + sorted[middle] + }; + format!( + "{{\"min_ms\":{},\"median_ms\":{},\"max_ms\":{},\"samples_ms\":{:?}}}", + sorted[0], + median, + sorted[sorted.len() - 1], + samples + ) +} + +fn median(variant: &Variant) -> f64 { + let mut sorted = variant.samples_ms.clone(); + sorted.sort_unstable_by(f64::total_cmp); + let middle = sorted.len() / 2; + if sorted.len().is_multiple_of(2) { + (sorted[middle - 1] + sorted[middle]) / 2.0 + } else { + sorted[middle] + } +} + +fn measure( + packed: &PackedAnalysis, + reference: &[u32], + repeats: usize, + name: &'static str, + threads: usize, + solve: fn(&PackedAnalysis) -> Vec, +) -> Result { + let first_start = Instant::now(); + let first_result = solve(packed); + let first_ms = first_start.elapsed().as_secs_f64() * 1_000.0; + if first_result != reference { + return Err(format!( + "CPU {name} output differs from the serial worklist" + )); + } + let mut samples_ms = Vec::with_capacity(repeats); + for _ in 0..repeats { + let start = Instant::now(); + let result = solve(packed); + samples_ms.push(start.elapsed().as_secs_f64() * 1_000.0); + // Equality checking is deliberately outside the solve timer, as in + // CUDA. Output allocation, scratch reset and thread creation are in it. + if result != reference { + return Err(format!("CPU {name} output changed between repetitions")); + } + } + Ok(Variant { + name, + threads, + first_ms, + samples_ms, + }) +} + +fn run() -> Result<(), String> { + let arguments = env::args_os().skip(1).collect::>(); + if arguments.len() == 1 && arguments[0] == "--help" { + println!("Usage: gpu-cpu-liveness INPUT.bin OUTPUT.bin [repeats]"); + return Ok(()); + } + if !(2..=3).contains(&arguments.len()) { + return Err("Usage: gpu-cpu-liveness INPUT.bin OUTPUT.bin [repeats]".into()); + } + let repeats = if let Some(raw) = arguments.get(2) { + raw.to_str() + .ok_or("Invalid repetition count")? + .parse::() + .map_err(|_| "Invalid repetition count")? + } else { + 15 + }; + if !(1..=10_000).contains(&repeats) { + return Err("Repetitions must be in 1..=10000".into()); + } + let input_path = fs::canonicalize(&arguments[0]) + .map_err(|error| format!("Cannot resolve input: {error}"))?; + if fs::canonicalize(&arguments[1]).ok().as_ref() == Some(&input_path) { + return Err("Input and output must be different files".into()); + } + let bytes = fs::read(&arguments[0]).map_err(|error| format!("Cannot read input: {error}"))?; + let initialize = Instant::now(); + let (packed, input_decode_ms, predecessor_prepare_ms) = glc::decode_timed(&bytes)?; + let initialization_ms = initialize.elapsed().as_secs_f64() * 1_000.0; + let reference = solve_cpu_serial(&packed); + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + let function_worker_limit = threads.min(packed.functions.len()); + let functions_per_worker = packed.functions.len().div_ceil(function_worker_limit); + let function_workers = packed.functions.len().div_ceil(functions_per_worker); + let word_tasks = packed + .functions + .iter() + .map(|f| f.words.div_ceil(4)) + .sum::(); + let variants = [ + measure(&packed, &reference, repeats, "serial", 1, solve_cpu_serial)?, + measure( + &packed, + &reference, + repeats, + "function_parallel", + function_workers, + solve_cpu_parallel, + )?, + measure( + &packed, + &reference, + repeats, + "word_parallel", + threads.min(word_tasks), + solve_cpu_word_parallel, + )?, + ]; + let selected = variants + .iter() + .min_by(|left, right| median(left).total_cmp(&median(right))) + .unwrap(); + let mut result = fs::File::create(&arguments[1]) + .map_err(|error| format!("Cannot create output: {error}"))?; + result + .write_all(b"GLR1") + .map_err(|error| error.to_string())?; + result + .write_all(&(reference.len() as u32).to_le_bytes()) + .map_err(|error| error.to_string())?; + for cell in &reference { + result + .write_all(&cell.to_le_bytes()) + .map_err(|error| error.to_string())?; + } + let variant_json = variants.iter().map(|variant| format!( + "{{\"name\":\"{}\",\"threads\":{},\"first_pass_end_to_end_ms\":{},\"repeat_outputs_equal\":true,\"warm_pass_end_to_end_ms\":{}}}", + variant.name, variant.threads, variant.first_ms, statistics(&variant.samples_ms) + )).collect::>().join(","); + println!(concat!( + "{{\"backend\":\"cpu-native\",\"functions\":{},\"blocks\":{},\"cells\":{},", + "\"groups\":{},\"max_blocks\":{},\"threads\":{},\"repeats\":{},", + "\"selected_variant\":\"{}\",\"repeat_outputs_equal\":true,\"initialization_ms\":{},", + "\"input_decode_ms\":{},\"predecessor_prepare_ms\":{},\"first_pass_end_to_end_ms\":{},", + "\"warm_pass_end_to_end_ms\":{},\"variants\":[{}],", + "\"timing_scope\":\"Warm elapsed includes solve, output and scratch allocation/reset, result materialization, and worker thread creation/join where used; excludes input file IO, GLC1 decoding, predecessor CSR preparation, output file IO, and cross-variant verification.\"}}" + ), packed.functions.len(), packed.block_count(), packed.total_cells, + packed.groups.len(), packed.max_blocks(), threads, repeats, selected.name, + initialization_ms, input_decode_ms, predecessor_prepare_ms, selected.first_ms, + statistics(&selected.samples_ms), variant_json); + Ok(()) +} + +fn json_string(value: &str) -> String { + let mut output = String::from("\""); + for character in value.chars() { + match character { + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + control if control.is_control() => { + output.push_str(&format!("\\u{:04x}", control as u32)) + } + normal => output.push(normal), + } + } + output.push('"'); + output +} + +fn main() { + if let Err(error) = run() { + eprintln!("{{\"error\":{}}}", json_string(&error)); + std::process::exit(1); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-emit-module.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-emit-module.rs new file mode 100644 index 00000000000..842198f1228 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-emit-module.rs @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use gpu_rust_compiler::{ + codegen_service::*, leaf_packet, machine_codegen, native_codegen::NativeExecutor, +}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, env, fs, io::Write, path::PathBuf, time::Instant}; +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +// Own only files successfully created by this invocation. Any later error must +// discard both outputs so an object cannot outlive its qualification receipt. +struct Publication { + files: Vec, + committed: bool, +} +impl Publication { + fn new() -> Self { + Self { + files: Vec::new(), + committed: false, + } + } + fn write(&mut self, path: &PathBuf, bytes: &[u8]) -> Result<(), String> { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| e.to_string())?; + self.files.push(path.clone()); + file.write_all(bytes).map_err(|e| e.to_string())?; + file.sync_all().map_err(|e| e.to_string()) + } +} +impl Drop for Publication { + fn drop(&mut self) { + if !self.committed { + for path in &self.files { + let _ = fs::remove_file(path); + } + } + } +} +fn run() -> Result<(), String> { + let mut options = BTreeMap::new(); + let mut args = env::args().skip(1); + while let Some(key) = args.next() { + if key == "--help" { + println!("gpu-emit-module --input GEM1 --output OBJECT --backend metal|cuda --library FILE [--shader FILE] [--device N] --report JSON\nStrict LLVM leaf-module lowering only; other Rust modules require explicit fallback."); + return Ok(()); + } + if ![ + "--input", + "--output", + "--backend", + "--library", + "--shader", + "--device", + "--report", + ] + .contains(&key.as_str()) + { + return Err(format!("unknown option{key}")); + } + let value = args + .next() + .ok_or_else(|| format!("missing value for{key}"))?; + if options.insert(key.clone(), value).is_some() { + return Err(format!("duplicate option{key}")); + } + } + let required = |key: &str| { + options + .get(key) + .cloned() + .ok_or_else(|| format!("missing{key}")) + }; + let input = PathBuf::from(required("--input")?); + let output = PathBuf::from(required("--output")?); + let report = PathBuf::from(required("--report")?); + if output.exists() || report.exists() || output == report { + return Err("output/report must be distinct new files".into()); + } + let backend = match required("--backend")?.as_str() { + "metal" => GpuBackend::Metal, + "cuda" => GpuBackend::Cuda, + _ => return Err("backend must be metal or cuda".into()), + }; + let library = PathBuf::from(required("--library")?); + let shader = options.get("--shader").map(PathBuf::from); + let device = options + .get("--device") + .map_or(Ok(0), |s| s.parse::().map_err(|_| "invalid device"))?; + let all = Instant::now(); + let data = fs::read(&input).map_err(|e| e.to_string())?; + let decoded = leaf_packet::decode(&data)?; + let batch = decoded.batch; + let target = decoded.target; + let init = Instant::now(); + let executor = NativeExecutor::new(backend, target, &library, shader.as_deref(), device)?; + let stats = executor.stats(); + let session = BuildSession::new( + SessionConfig { + cpu_credit_limit: 1, + ..Default::default() + }, + vec![Box::new(executor)], + ) + .map_err(|e| e.to_string())?; + let client = session + .register_client("rust-leaf-module", 1) + .map_err(|e| e.to_string())?; + let initialization_ms = init.elapsed().as_secs_f64() * 1000.; + let ticket = session + .submit( + &client, + batch.clone(), + SubmitOptions { + placement: Placement::RequireGpu(backend), + ..Default::default() + }, + ) + .map_err(|e| e.to_string())?; + let receipt = ticket.wait().map_err(|e| e.to_string())?; + if !session.is_current(&receipt) { + return Err("stale GPU module cannot be published".into()); + } + // Verification is included in total elapsed time, never hidden as a GPU win. + let oracle = machine_codegen::cpu_emit(&batch, target)?; + let mut expected = oracle.functions; + expected.sort_by_key(|f| f.symbol); + let mut actual = receipt.output.functions.clone(); + actual.sort_by_key(|f| f.symbol); + if actual != expected { + return Err("GPU module differs from CPU encoding oracle".into()); + } + let bytes = machine_codegen::write_object(&batch, target, &receipt.output)?; + let file = object::File::parse(bytes.as_slice()).map_err(|e| e.to_string())?; + use object::Object as _; + if file.kind() != object::ObjectKind::Relocatable { + return Err("GPU output is not a relocatable object".into()); + } + let mut publication = Publication::new(); + publication.write(&output, &bytes)?; + let native_stats = stats.lock().unwrap().clone(); + let dispatches = native_stats + .iter() + .map(|s| s.kernel_dispatches) + .sum::(); + let llvm_input_target = decoded.llvm_target; + let json = serde_json::json!({"schema":"gpu-emit-module","version":1,"backend":match backend{GpuBackend::Metal=>"metal",GpuBackend::Cuda=>"cuda"},"physical_gpu":receipt.execution.is_physical_gpu(),"device_id":receipt.route.device_id,"kernel_dispatches":dispatches,"kernel_output_used":true,"gpu_executed":true,"performance_speedup_demonstrated":false,"scope":"strict whole LLVM leaf module to native object; CPU generates required unwind metadata; unsupported Rust modules must use explicit fallback","input_packet_sha256":hash(&data),"object_bytes":bytes.len(),"object_sha256":hash(&bytes),"function_count":batch.functions().len(),"target":batch.identity().target_abi,"llvm_input_target":llvm_input_target,"total_elapsed_ms":all.elapsed().as_secs_f64()*1000.,"initialization_ms":initialization_ms,"cpu_byte_oracle_verified":true,"unwind_metadata_required":batch.abis().iter().any(|a|a.requires_unwind),"native_library_sha256":hash(&fs::read(&library).map_err(|e|e.to_string())?),"native_stats":native_stats,"timing_scope":"includes packet decode/validation, GPU setup, GPU work/wait, CPU byte oracle, unwind/object serialization and output write; no original LLVM native object emission"}); + let text = serde_json::to_vec_pretty(&json).map_err(|e| e.to_string())?; + publication.write(&report, &text)?; + publication.committed = true; + println!("{}", String::from_utf8_lossy(&text)); + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("gpu-emit-module: {error}"); + std::process::exit(1); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-native-benchmark.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-native-benchmark.rs new file mode 100644 index 00000000000..41922468d58 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-native-benchmark.rs @@ -0,0 +1,571 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Persistent native workpack driver. Python is only the external controller. + +#[allow(dead_code, clippy::needless_range_loop)] +#[path = "../analysis.rs"] +mod analysis; +#[allow(dead_code)] +#[path = "../cuda.rs"] +mod cuda; +#[allow(dead_code)] +#[path = "../executor.rs"] +mod executor; +#[path = "../glc.rs"] +mod glc; +#[allow(dead_code)] +#[path = "../ir.rs"] +mod ir; +#[allow(dead_code)] +#[path = "../routing.rs"] +mod routing; + +use analysis::{select_packed, PackedAnalysis}; +use cuda::{Algorithm, Context}; +use executor::{AnalysisSelection, CpuExecutor, CpuMode}; +use std::{ + env, fs, + io::{self, BufRead, Write}, + path::{Path, PathBuf}, + sync::Arc, + time::Instant, +}; + +fn elapsed(start: Instant) -> f64 { + start.elapsed().as_secs_f64() * 1_000.0 +} + +fn quote(value: &str) -> String { + let mut output = String::from("\""); + for character in value.chars() { + match character { + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + c if c.is_control() => output.push_str(&format!("\\u{:04x}", c as u32)), + c => output.push(c), + } + } + output.push('"'); + output +} + +fn cpu_mode(value: &str) -> Result, String> { + match value.split('@').next().unwrap_or(value) { + "auto" => Ok(None), + "serial" => Ok(Some(CpuMode::Serial)), + "function_pool" | "function_parallel" => Ok(Some(CpuMode::Functions)), + "word_pool" | "word_parallel" => Ok(Some(CpuMode::Words)), + _ => Err("CPU mode must be auto, serial, function_pool or word_pool".into()), + } +} + +fn algorithm(value: &str) -> Result { + match value { + "dense" => Ok(Algorithm::Dense), + "sparse" => Ok(Algorithm::Sparse), + _ => Err("CUDA algorithm must be dense or sparse".into()), + } +} + +fn write_result(path: &Path, words: &[u32]) -> Result<(), String> { + let mut bytes = Vec::with_capacity(8 + words.len() * 4); + bytes.extend_from_slice(b"GLR1"); + bytes.extend_from_slice(&(words.len() as u32).to_le_bytes()); + for word in words { + bytes.extend_from_slice(&word.to_le_bytes()); + } + fs::write(path, bytes).map_err(|error| format!("Cannot write output: {error}")) +} + +struct Session { + pool: CpuExecutor, + pool_initialization_ms: f64, + library: Option, + context: Option, + decoded: Option<(Vec, Arc)>, + resident_cuda: Option>, + profile_path: Option, + requests: usize, + context_creations: usize, +} + +impl Session { + fn new(library: Option, profile_path: Option, workers: usize) -> Self { + let start = Instant::now(); + let pool = CpuExecutor::new(workers); + Self { + pool, + pool_initialization_ms: elapsed(start), + library, + context: None, + decoded: None, + resident_cuda: None, + profile_path, + requests: 0, + context_creations: 0, + } + } + + fn context(&mut self) -> Result<&mut Context, String> { + if self.context.is_none() { + let path = self + .library + .as_ref() + .ok_or("CUDA library was not specified")?; + self.context = Some(Context::new(path)?); + self.context_creations += 1; + } + Ok(self.context.as_mut().unwrap()) + } + + fn request(&mut self, fields: &[&str]) -> Result { + if fields.first() == Some(&"profile") { + return self.record_profile(fields); + } + if fields.len() != 7 { + return Err( + "Expected seven TAB fields: id backend algorithm input output residency cpu_mode" + .into(), + ); + } + let start = Instant::now(); + let (id, backend, algorithm_name, input, output, residency, cpu_name) = ( + fields[0], fields[1], fields[2], fields[3], fields[4], fields[5], fields[6], + ); + if !matches!( + backend, + "cpu" | "cuda" | "hybrid" | "forced-hybrid" | "calibrate" | "abi-check" + ) { + return Err("Unknown backend".into()); + } + if !matches!(residency, "update" | "resident") { + return Err("Residency must be update or resident".into()); + } + let mut algorithm = algorithm(algorithm_name)?; + let mut mode = cpu_mode(cpu_name)?; + let input_path = fs::canonicalize(input).map_err(|error| error.to_string())?; + if fs::canonicalize(output).ok().as_ref() == Some(&input_path) { + return Err("Input and output must differ".into()); + } + let bytes = fs::read(&input_path).map_err(|error| format!("Cannot read input: {error}"))?; + let mut decode_ms = 0.0; + let mut predecessor_ms = 0.0; + let packed = if residency == "resident" { + let (previous, packed) = self + .decoded + .as_ref() + .ok_or("No decoded input is resident")?; + if previous != &bytes { + return Err("Resident request differs from resident input; use update".into()); + } + packed.clone() + } else { + let (decoded, input_ms, reverse_ms) = glc::decode_timed(&bytes)?; + decode_ms = input_ms; + predecessor_ms = reverse_ms; + let packed = Arc::new(decoded); + self.decoded = Some((bytes.clone(), packed.clone())); + packed + }; + let mut calibration_json = "null".to_string(); + if let Some((_, limit)) = cpu_name.split_once('@') { + let limit = limit + .parse() + .map_err(|_| "Invalid active CPU worker limit")?; + self.pool.set_worker_limit( + &packed, + mode.ok_or("Active worker limit needs explicit mode")?, + limit, + )?; + } + if backend == "calibrate" { + let calibration = self.pool.calibrate(packed.clone(), 7)?; + let variants = calibration + .variants + .iter() + .map(|variant| { + format!( + "{{\"name\":{},\"workers\":{},\"median_ms\":{},\"p90_ms\":{},\"samples_ms\":{:?}}}", + quote(variant.mode.name()), + variant.workers, + variant.median_ms, + variant.p90_ms, + variant.samples_ms + ) + }) + .collect::>() + .join(","); + calibration_json = format!( + "{{\"selected_mode\":{},\"selected_workers\":{},\"repeats\":{},\"variants\":[{}]}}", + quote(calibration.selected_mode.name()), + calibration.selected_workers, + calibration.repeats, + variants + ); + mode = Some(calibration.selected_mode); + } + let analysis_start = Instant::now(); + let mut gpu_stats = "null".to_string(); + let mut gpu_functions = 0; + let mut cpu_functions = packed.functions.len(); + let mut packing_ms = 0.0; + let mut scatter_ms = 0.0; + let mut cpu_ms = 0.0; + let mut submitted_before_cpu = false; + let mut cpu_while_pending = false; + let mut event_pending_before_cpu = false; + let mut event_pending_after_cpu = false; + let mut pending_rejected = false; + let mut snapshot_checked = false; + let mut routing_reason = "Explicit CPU execution".to_string(); + let result = if matches!(backend, "cpu" | "calibrate") { + let cpu_start = Instant::now(); + let solved = match mode { + Some(mode) => self.pool.solve_mode(packed.clone(), mode), + None => self.pool.solve(packed.clone()), + }; + cpu_ms = elapsed(cpu_start); + solved + } else if matches!(backend, "cuda" | "abi-check") { + if residency == "resident" && self.resident_cuda.as_ref() != Some(&bytes) { + return Err( + "CUDA resident request differs from last GPU upload; use update".into(), + ); + } + let context = self.context()?; + if backend == "abi-check" { + let mut submitted = (*packed).clone(); + context.submit(&submitted, algorithm)?; + if context.submit(&submitted, algorithm).is_ok() { + return Err("Native adapter accepted a second pending submission".into()); + } + pending_rejected = true; + // H2D may still run when the caller changes or drops its input. + submitted.uses.fill(0); + submitted.phi_out.fill(0); + submitted.defs.fill(u32::MAX); + snapshot_checked = true; + } else if residency == "resident" { + context.submit_resident(algorithm)?; + } else { + context.submit(&packed, algorithm)?; + } + let (solved, stats) = context.finish()?; + gpu_stats = stats_json(&stats); + gpu_functions = packed.functions.len(); + cpu_functions = 0; + self.resident_cuda = Some(bytes); + routing_reason = "Explicit CUDA execution".into(); + solved + } else { + // Mixed execution always counts packing and transfer; replaying a + // cached subset would conceal costs and changing routing decisions. + if residency != "update" { + return Err("Hybrid requires update residency".into()); + } + let (cpu_indices, gpu_indices, reason, routed_mode, routed_algorithm) = + self.route(&packed, backend)?; + if backend == "hybrid" { + mode = Some(routed_mode); + algorithm = routed_algorithm; + } + routing_reason = reason; + cpu_functions = cpu_indices.len(); + gpu_functions = gpu_indices.len(); + let mut result = vec![0; packed.total_cells]; + if gpu_indices.is_empty() { + let cpu_start = Instant::now(); + result = match mode { + Some(mode) => self.pool.solve_mode(packed.clone(), mode), + None => self.pool.solve(packed.clone()), + }; + cpu_ms = elapsed(cpu_start); + } else { + let packing_start = Instant::now(); + let gpu_pack = select_packed(&packed, &gpu_indices)?; + let cpu_selection = AnalysisSelection::new(&packed, &cpu_indices)?; + packing_ms = elapsed(packing_start); + self.context()?.submit(&gpu_pack, algorithm)?; + submitted_before_cpu = !cpu_indices.is_empty(); + event_pending_before_cpu = self.context()?.is_pending()?; + let cpu_start = Instant::now(); + let cpu_output = self.pool.solve_selection( + packed.clone(), + &cpu_selection, + mode.unwrap_or(CpuMode::Serial), + ); + cpu_ms = elapsed(cpu_start); + cpu_while_pending = !cpu_indices.is_empty(); + event_pending_after_cpu = self.context()?.is_pending()?; + let (gpu_output, stats) = self.context()?.finish()?; + gpu_stats = stats_json(&stats); + let scatter_start = Instant::now(); + cpu_output.scatter_into(&mut result)?; + analysis::scatter(&gpu_pack, &gpu_output, &packed, &mut result)?; + scatter_ms = elapsed(scatter_start); + self.resident_cuda = None; + } + result + }; + let analysis_ms = elapsed(analysis_start); + write_result(Path::new(output), &result)?; + self.requests += 1; + let total_ms = elapsed(start); + let capabilities = self + .context + .as_ref() + .map(|context| capabilities_json(&context.capabilities())) + .unwrap_or_else(|| "null".to_string()); + Ok(format!(concat!( + "{{\"status\":\"ok\",\"request_id\":{},\"backend\":{},\"algorithm\":{},", + "\"residency\":{},\"cpu_mode\":{},\"process_id\":{},\"request_number\":{},", + "\"functions\":{},\"blocks\":{},\"cells\":{},\"actual_gpu_functions\":{},", + "\"actual_cpu_functions\":{},\"input_decode_ms\":{},\"predecessor_prepare_ms\":{},", + "\"packing_ms\":{},\"scatter_ms\":{},\"cpu_solve_ms\":{},\"analysis_ms\":{},", + "\"total_recurrent_ms\":{},\"cpu_pool_initialization_ms\":{},\"cpu_pool_workers\":{},", + "\"cpu_pool_jobs_completed\":{},\"cpu_pool_submissions\":{},\"cpu_pool_reused\":{},\"cuda_context_creations\":{},\"gpu_stats\":{},", + "\"capabilities\":{},\"cpu_calibration\":{},\"routing_reason\":{},", + "\"gpu_submitted_before_cpu\":{},\"cpu_work_before_gpu_finish\":{},", + "\"second_pending_submit_rejected\":{},\"input_snapshot_ownership_checked\":{},", + "\"gpu_event_pending_before_cpu\":{},\"gpu_event_pending_after_cpu\":{},", + "\"overlap_scope\":\"CUDA completion-event state is queried before/after independent CPU work; no device trace is collected to claim kernel/CPU overlap duration.\"}}" + ), quote(id), quote(backend), quote(match algorithm { Algorithm::Dense => "dense", Algorithm::Sparse => "sparse" }), + quote(residency), quote(mode.map_or("auto", CpuMode::name)), + std::process::id(), self.requests, packed.functions.len(), packed.block_count(), packed.total_cells, + gpu_functions, cpu_functions, decode_ms, predecessor_ms, packing_ms, scatter_ms, cpu_ms, + analysis_ms, total_ms, self.pool_initialization_ms, self.pool.worker_count(), self.pool.jobs_completed(), + self.pool.pool_submissions(), self.pool.pool_reused(), + self.context_creations, gpu_stats, capabilities, calibration_json, quote(&routing_reason), + submitted_before_cpu, cpu_while_pending, pending_rejected, snapshot_checked, + event_pending_before_cpu, event_pending_after_cpu)) + } + + fn record_profile(&mut self, fields: &[&str]) -> Result { + if !(9..=10).contains(&fields.len()) { + return Err("Expected profile,input,cpu_mode,algorithm,four sample arrays,output[,candidate indices]".into()); + } + let bytes = fs::read(fields[1]).map_err(|error| error.to_string())?; + let (packed, _, _) = glc::decode_timed(&bytes)?; + let mode = cpu_mode(fields[2])?.ok_or("Profile requires an explicit CPU mode")?; + let alg = match fields[3] { + "dense" => routing::GpuAlgorithm::Dense, + "sparse" => routing::GpuAlgorithm::Sparse, + _ => return Err("Unknown profile algorithm".into()), + }; + let samples = fields[4..8] + .iter() + .map(|raw| { + raw.split(',') + .map(|value| { + value + .parse::() + .map_err(|_| "Invalid sample value".to_string()) + }) + .collect::, _>>() + }) + .collect::, _>>()?; + let workers = self.pool.worker_count(); + let fingerprint = self.context()?.capabilities().fingerprint(workers); + let identity = routing::hardware_identity(&fingerprint, workers); + let path = Path::new(fields[8]); + let mut profile = if path.exists() { + routing::RouteProfile::load(path)? + } else { + routing::RouteProfile::new(identity.clone(), routing::Objective::Latency) + }; + if profile.hardware_identity != identity { + return Err("Existing profile hardware identity differs".into()); + } + let active_workers = self.pool.worker_limit(&packed, mode); + if let Some(indices) = fields.get(9) { + let indices = indices + .split(',') + .map(|value| { + value + .parse::() + .map_err(|_| "Invalid candidate index".to_string()) + }) + .collect::, _>>()?; + let candidate = select_packed(&packed, &indices)?; + profile.record_with_context_and_workers( + &candidate, + &packed, + mode, + active_workers, + alg, + &samples[0], + &samples[1], + &samples[2], + &samples[3], + )?; + } else { + profile.record_with_context_and_workers( + &packed, + &packed, + mode, + active_workers, + alg, + &samples[0], + &samples[1], + &samples[2], + &samples[3], + )?; + } + profile.save(path)?; + let plan = profile.plan(&packed, &identity, routing::Objective::Latency); + Ok(format!("{{\"status\":\"ok\",\"process_id\":{},\"profile_path\":{},\"gpu_functions_admitted\":{},\"cpu_functions\":{},\"reason\":{}}}", + std::process::id(), quote(fields[8]), plan.gpu_indices.len(), plan.cpu_indices.len(), quote(&plan.reason))) + } + + #[allow(clippy::type_complexity)] + fn route( + &mut self, + packed: &PackedAnalysis, + backend: &str, + ) -> Result<(Vec, Vec, String, CpuMode, Algorithm), String> { + if backend == "forced-hybrid" { + let (gpu, cpu): (Vec<_>, Vec<_>) = packed + .functions + .iter() + .partition(|function| function.block_count <= 64); + return Ok(( + cpu.iter().map(|f| f.module_index).collect(), + gpu.iter().map(|f| f.module_index).collect(), + "Forced split for independent-subset correctness; not a speedup policy".into(), + CpuMode::Serial, + Algorithm::Dense, + )); + } + if let Some(path) = self.profile_path.clone() { + let profile = routing::RouteProfile::load(&path)?; + let workers = self.pool.worker_count(); + let fingerprint = self.context()?.capabilities().fingerprint(workers); + let identity = routing::hardware_identity(&fingerprint, workers); + let plan = profile.plan(packed, &identity, routing::Objective::Latency); + if let Some(limit) = plan.cpu_worker_limit { + self.pool.set_worker_limit(packed, plan.cpu_mode, limit)?; + } + let algorithm = match plan.algorithm { + routing::GpuAlgorithm::Dense => Algorithm::Dense, + routing::GpuAlgorithm::Sparse => Algorithm::Sparse, + }; + return Ok(( + plan.cpu_indices, + plan.gpu_indices, + plan.reason, + plan.cpu_mode, + algorithm, + )); + } + Ok(( + packed.functions.iter().map(|f| f.module_index).collect(), + vec![], + "No validated route profile; conservative CPU fallback".into(), + CpuMode::Serial, + Algorithm::Dense, + )) + } +} + +fn stats_json(stats: &cuda::Stats) -> String { + stats.json() +} + +fn capabilities_json(cap: &cuda::Capabilities) -> String { + cap.json() +} + +fn run() -> Result<(), String> { + let arguments = env::args().skip(1).collect::>(); + let mut library = None; + let mut profile = None; + let mut workers = 0; + let mut index = 0; + let mut serve = false; + while index < arguments.len() { + match arguments[index].as_str() { + "--serve" => serve = true, + "--cuda-library" | "--route-profile" | "--threads" => { + let value = arguments.get(index + 1).ok_or("Option requires a value")?; + match arguments[index].as_str() { + "--cuda-library" => library = Some(PathBuf::from(value)), + "--route-profile" => profile = Some(PathBuf::from(value)), + _ => workers = value.parse().map_err(|_| "Invalid thread count")?, + } + index += 1; + } + "--help" => { + println!("Usage: gpu-native-benchmark --serve [--cuda-library LIB] [--route-profile CSV] [--threads N]"); + println!("stdin: id TAB backend TAB algorithm TAB input TAB output TAB update|resident TAB auto|serial|function_parallel|word_parallel"); + return Ok(()); + } + _ => return Err(format!("Unknown argument: {}", arguments[index])), + } + index += 1; + } + if !serve || workers > 256 { + return Err( + "--serve is required; thread count must be 0..=256 (0 uses all available CPUs)".into(), + ); + } + let mut session = Session::new(library, profile, workers); + for line in io::stdin().lock().lines() { + let line = line.map_err(|error| error.to_string())?; + if line == "quit" { + break; + } + let fields = line.split('\t').collect::>(); + let response = match session.request(&fields) { + Ok(value) => value, + Err(error) => format!( + "{{\"status\":\"error\",\"request_id\":{},\"error\":{},\"process_id\":{}}}", + quote(fields.first().copied().unwrap_or("")), + quote(&error), + std::process::id() + ), + }; + println!("{response}"); + io::stdout().flush().map_err(|error| error.to_string())?; + } + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("{{\"error\":{}}}", quote(&error)); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn subset_pack_remaps_edges_and_scatter_without_leaking_function_ownership() { + let words = [ + 2u32, 3, 3, 2, 2, 1, 0, 2, 0, 1, 0, 2, 1, 2, 1, 0, 0, 1, 1, 1, 1, 0, 1, 2, 0, 0, 0, 0, + 0, 0, + ]; + let mut bytes = b"GLC1".to_vec(); + bytes.extend(words.iter().flat_map(|word| word.to_le_bytes())); + let original = glc::decode(&bytes).unwrap(); + let subset = select_packed(&original, &[1]).unwrap(); + assert_eq!(subset.functions[0].module_index, 1); + assert_eq!(subset.functions[0].block_base, 0); + assert!(subset.successors.is_empty()); + let mut result = vec![0; 3]; + analysis::scatter( + &subset, + &analysis::solve_cpu_serial(&subset), + &original, + &mut result, + ) + .unwrap(); + assert_eq!(result, vec![0, 0, 2]); + assert!(select_packed(&original, &[1, 1]).is_err()); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-reference-programs.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-reference-programs.rs new file mode 100644 index 00000000000..ff8248ecbef --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-reference-programs.rs @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Build immutable standard-Rust output references before entering the CUDA image. + +use std::{env, fs, path::Path, process::Command}; + +const DEMO: &str = include_str!("../../../fixtures/compiler_demo.rs"); +const EDGES: &str = r#"// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +fn choose(value: i64, flag: bool) -> i64 { + if flag && value > 0 { return value * 3; } + return value - 5; +} +fn main() -> i64 { + let mut sum: i64 = 0; + let mut index: i64 = 0; + while index < 20 { + sum = sum + choose(index, index % 2 == 0); + index = index + 1; + } + if false && 1 / 0 > 0 { return 999; } + if true || 1 / 0 > 0 { sum = sum + 7; } + return (sum << 67) ^ (-41 >> 65); +} +"#; +const CHANGED: &str = r#"// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +fn main() -> i64 { + let mut index: i64 = 9; + let mut total: i64 = 0; + while index > 0 { + if index % 2 == 0 { total = total + index * 7; } + else { total = total - index; } + index = index - 1; + } + return total; +} +"#; + +fn run() -> Result<(), String> { + let arguments = env::args_os().skip(1).collect::>(); + if arguments.len() != 1 { + return Err("Usage: gpu-reference-programs OUTPUT_DIRECTORY (requires rustc)".into()); + } + let directory = Path::new(&arguments[0]); + fs::create_dir_all(directory).map_err(|error| error.to_string())?; + let identity = Command::new("rustc") + .arg("--version") + .arg("--verbose") + .output() + .map_err(|error| format!("Cannot run rustc: {error}"))?; + if !identity.status.success() { + return Err(String::from_utf8_lossy(&identity.stderr).into_owned()); + } + fs::write(directory.join("rustc-version.txt"), &identity.stdout) + .map_err(|error| error.to_string())?; + for (name, source) in [("demo", DEMO), ("semantics", EDGES), ("changed", CHANGED)] { + let source_path = directory.join(format!("{name}.rs")); + let harness_path = directory.join(format!("{name}-reference.rs")); + let executable = directory.join(format!("{name}-reference")); + fs::write(&source_path, source).map_err(|error| error.to_string())?; + // Runtime shifts and short-circuit arithmetic avoid Rust's compile-time + // constant-overflow lint; both compilers use wrapping arithmetic here. + let harness = source.replace("fn main() -> i64", "fn program_main() -> i64"); + fs::write( + &harness_path, + format!("{harness}\nfn main() {{ println!(\"{{}}\", program_main()); }}\n"), + ) + .map_err(|error| error.to_string())?; + let compiled = Command::new("rustc") + .args([ + "--edition=2021", + "-Awarnings", + "-Aunconditional_panic", + "-Aarithmetic_overflow", + ]) + .args(["-C", "overflow-checks=off", "-C", "opt-level=0"]) + .arg(&harness_path) + .arg("-o") + .arg(&executable) + .output() + .map_err(|error| error.to_string())?; + if !compiled.status.success() { + return Err(format!( + "{name}: {}", + String::from_utf8_lossy(&compiled.stderr) + )); + } + let result = Command::new(&executable) + .output() + .map_err(|error| error.to_string())?; + if !result.status.success() || !result.stderr.is_empty() { + return Err(format!("{name}: reference executable failed")); + } + fs::write( + directory.join(format!("{name}.expected.txt")), + &result.stdout, + ) + .map_err(|error| error.to_string())?; + } + fs::write(directory.join("programs.txt"), "demo\nsemantics\nchanged\n") + .map_err(|error| error.to_string())?; + println!("Built standard-Rust references: {}", directory.display()); + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("{error}"); + std::process::exit(1); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bin/gpu-verify-pipeline.rs b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-verify-pipeline.rs new file mode 100644 index 00000000000..3377c24a501 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bin/gpu-verify-pipeline.rs @@ -0,0 +1,508 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{json, Value}; +use std::{ + env, fs, + path::{Path, PathBuf}, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +const EXPECTED: &str = "rust-gpu-fixture: arithmetic=verified caught=true drops=1\n"; +const HELP: &str = + "Usage: gpu-verify-pipeline --gpu-cargo FILE --bitcode-wrapper FILE --controller FILE + --tpde-bridge FILE --llvm-llc FILE --gpu-emitter FILE --gpu-library FILE + --backend metal|cuda [--shader FILE] --target-root DIRECTORY --report FILE + [--fixture-manifest FILE] + [--reference-artifact FILE] [--max-size-ratio 1.5] [--size-basis unmodified|strip-debug] + +Creates an owned verification run, builds the full Rust fixture with required +physical GPU objects plus attributed CPU fallback, executes it, and checks an +identical cache-only build. This proves execution and cache behavior, not speedup."; + +struct Options { + fixture: Option, + cargo: PathBuf, + wrapper: PathBuf, + controller: PathBuf, + bridge: PathBuf, + llc: PathBuf, + emitter: PathBuf, + library: PathBuf, + shader: Option, + backend: String, + target_root: PathBuf, + report: PathBuf, + reference: Option, + ratio: String, + basis: String, +} + +fn options() -> Result, String> { + let mut args = env::args().skip(1); + let mut values = std::collections::BTreeMap::new(); + while let Some(arg) = args.next() { + if matches!(arg.as_str(), "--help" | "-h") { + println!("{HELP}"); + return Ok(None); + } + let arg = match arg.as_str() { + "--wrapper" => "--bitcode-wrapper".into(), + "--bridge" => "--tpde-bridge".into(), + "--llc" => "--llvm-llc".into(), + _ => arg, + }; + if !matches!( + arg.as_str(), + "--gpu-cargo" + | "--bitcode-wrapper" + | "--controller" + | "--tpde-bridge" + | "--llvm-llc" + | "--gpu-emitter" + | "--gpu-library" + | "--shader" + | "--backend" + | "--target-root" + | "--report" + | "--fixture-manifest" + | "--reference-artifact" + | "--max-size-ratio" + | "--size-basis" + ) { + return Err(format!("Unknown option {arg}")); + } + let value = args + .next() + .ok_or_else(|| format!("Missing value for {arg}"))?; + if values.insert(arg.clone(), value).is_some() { + return Err(format!("Repeated option {arg}")); + } + } + let path = |name: &str| -> Result { + values + .get(name) + .map(PathBuf::from) + .ok_or_else(|| format!("Missing {name}\n{HELP}")) + }; + let backend = values.get("--backend").ok_or("Missing --backend")?.clone(); + if !matches!(backend.as_str(), "metal" | "cuda") { + return Err("Backend must be metal or cuda".into()); + } + let shader = values.get("--shader").map(PathBuf::from); + if backend == "metal" && shader.is_none() { + return Err("Metal verification requires --shader".into()); + } + let reference = values.get("--reference-artifact").map(PathBuf::from); + if values.contains_key("--max-size-ratio") && reference.is_none() { + return Err("--max-size-ratio requires --reference-artifact".into()); + } + let basis = values + .get("--size-basis") + .cloned() + .unwrap_or_else(|| "unmodified".into()); + if !matches!(basis.as_str(), "unmodified" | "strip-debug") { + return Err("Size basis must be unmodified or strip-debug".into()); + } + Ok(Some(Options { + fixture: values.get("--fixture-manifest").map(PathBuf::from), + cargo: path("--gpu-cargo")?, + wrapper: path("--bitcode-wrapper")?, + controller: path("--controller")?, + bridge: path("--tpde-bridge")?, + llc: path("--llvm-llc")?, + emitter: path("--gpu-emitter")?, + library: path("--gpu-library")?, + shader, + backend, + target_root: path("--target-root")?, + report: path("--report")?, + reference, + ratio: values + .get("--max-size-ratio") + .cloned() + .unwrap_or_else(|| "1.5".into()), + basis, + })) +} + +fn validate_fresh_report(report: &Value) -> Result<(), String> { + if report["schema"] != "gpu-cargo-report" + || report["version"] != 1 + || report["status"] != "ok" + || report["object_backend"] != "gpu" + || report["quality"] != "fast" + || report["toolchain"] != "1.98.1" + { + return Err("Fresh build lacks a successful pinned GPU Cargo report".into()); + } + if report["gpu_executed"] != true + || report["current_run_gpu_qualified"] != true + || report["cache_only"] != false + || report["artifact"]["cargo_fresh"] != false + || report["performance_speedup_demonstrated"] != false + { + return Err( + "A cached, simulated, or unqualified build cannot establish fresh GPU execution".into(), + ); + } + let jobs = report["gpu_compilation_jobs"] + .as_u64() + .filter(|jobs| *jobs > 0) + .ok_or("No physical GPU module was compiled")?; + let evidence = &report["object_backend_receipts"]; + if evidence["physical_gpu_units"].as_u64() != Some(jobs) + || evidence["gpu_kernel_dispatches"].as_u64().unwrap_or(0) == 0 + || evidence["fallback_units"].as_u64().unwrap_or(0) == 0 + || evidence["gpu_devices"] + .as_object() + .is_none_or(|devices| devices.is_empty()) + { + return Err("Fresh GPU modules or attributed CPU fallback are missing".into()); + } + Ok(()) +} + +fn validate_cached_report(report: &Value, fresh: &Value) -> Result<(), String> { + if report["status"] != "ok" + || report["object_backend"] != "gpu" + || report["artifact"]["cargo_fresh"] != true + || report["cache_only"] != true + || report["gpu_executed"] != false + || report["current_run_gpu_qualified"] != false + || report["gpu_compilation_jobs"] != 0 + || report["performance_speedup_demonstrated"] != false + { + return Err( + "Unchanged build did not report qualified cache reuse without new GPU execution".into(), + ); + } + let evidence = &report["object_backend_receipts"]; + if evidence["physical_gpu_units"] != 0 + || evidence["translated_units"] != 0 + || evidence["gpu_kernel_dispatches"] != 0 + { + return Err("Cache-only build contains new object-emission work".into()); + } + let lineage = &report["cached_gpu_lineage"]; + if report["artifact"]["sha256"] != fresh["artifact"]["sha256"] + || report["artifact"]["path"] != fresh["artifact"]["path"] + || lineage["artifact_sha256"] != fresh["artifact"]["sha256"] + || lineage["physical_gpu_units"] != fresh["gpu_compilation_jobs"] + || lineage["qualified_receipts_directory"] != fresh["object_backend_receipts"]["directory"] + || lineage["qualified_receipt_hashes"] + .as_object() + .is_none_or(|hashes| hashes.is_empty()) + { + return Err( + "Cached artifact does not retain the fresh run's qualified receipt lineage".into(), + ); + } + Ok(()) +} + +fn verify_raw_receipts(report: &Value, backend: &str) -> Result<(u64, u64), String> { + let directory = Path::new( + report["object_backend_receipts"]["directory"] + .as_str() + .ok_or("No raw receipt directory")?, + ); + let mut gpu = 0u64; + let mut cpu = 0u64; + for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { + let path = entry.map_err(|e| e.to_string())?.path(); + if path.extension().is_none_or(|extension| extension != "json") { + continue; + } + let receipt: Value = serde_json::from_slice(&fs::read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if receipt["forwarded_passthrough"] == true { + continue; + } + if receipt["status"] != "success" + || receipt["frontend_bitcode_verified"] != true + || receipt["native_object_before_selected_emitter"] != false + || receipt["saved_link_without_recompilation"] != true + { + return Err(format!( + "Receipt does not prove bitcode-only frontend and saved linking: {}", + path.display() + )); + } + for unit in receipt["units"] + .as_array() + .ok_or("Receipt omitted object units")? + { + if unit["input_kind"] != "llvm-bitcode" + || unit["input_sha256"] + .as_str() + .is_none_or(|hash| hash.len() != 64) + || unit["object_sha256"] + .as_str() + .is_none_or(|hash| hash.len() != 64) + { + return Err("Object unit omitted immutable bitcode/native-object evidence".into()); + } + if unit["backend"] == backend { + if unit["physical_gpu"] != true + || unit["kernel_output_used"] != true + || unit["kernel_dispatches"].as_u64().unwrap_or(0) == 0 + || unit["device_id"].as_str().is_none_or(|id| id.is_empty()) + { + return Err("GPU object unit lacks physical execution evidence".into()); + } + gpu += 1; + } else { + if !matches!(unit["backend"].as_str(), Some("llvm" | "tpde")) + || unit["physical_gpu"] != false + || unit["kernel_output_used"] != false + || unit["fallback_reason"] + .as_str() + .is_none_or(|reason| reason.is_empty()) + { + return Err("CPU object fallback is missing its actual backend/reason".into()); + } + cpu += 1; + } + } + } + if gpu == 0 || cpu == 0 || report["gpu_compilation_jobs"].as_u64() != Some(gpu) { + return Err("Raw receipt GPU/CPU counts do not match the compiled fixture".into()); + } + Ok((gpu, cpu)) +} + +fn save_json(path: &Path, value: &Value) -> Result<(), String> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + fs::write( + path, + format!( + "{}\n", + serde_json::to_string(value).map_err(|e| e.to_string())? + ), + ) + .map_err(|e| e.to_string()) +} + +fn cargo_build( + options: &Options, + manifest: &Path, + target: &Path, + report_path: &Path, +) -> Result { + let mut command = Command::new(&options.cargo); + command + .args(["build", "--manifest-path"]) + .arg(manifest) + .args([ + "--package", + "gpu-rust-full-fixture", + "--bin", + "gpu-rust-full-fixture", + "--quality", + "fast", + "--object-backend", + "gpu", + "--placement", + &options.backend, + "--allow-backend-fallback", + "--offline", + "--bitcode-wrapper", + ]) + .arg(&options.wrapper); + for (name, path) in [ + ("--tpde-controller", options.controller.as_path()), + ("--tpde-bridge", options.bridge.as_path()), + ("--llvm-llc", options.llc.as_path()), + ("--gpu-emitter", options.emitter.as_path()), + ("--gpu-library", options.library.as_path()), + ("--target-root", target), + ("--report", report_path), + ] { + command.arg(name).arg(path); + } + command.args([ + "--gpu-backend", + &options.backend, + "--size-basis", + &options.basis, + ]); + if let Some(shader) = &options.shader { + command.arg("--gpu-shader").arg(shader); + } + if let Some(reference) = &options.reference { + command + .arg("--reference-artifact") + .arg(reference) + .args(["--max-size-ratio", &options.ratio]); + } + let result = command + .output() + .map_err(|e| format!("Cargo policy launch failed: {e}"))?; + fs::write(report_path.with_extension("stdout.log"), &result.stdout) + .map_err(|e| e.to_string())?; + fs::write(report_path.with_extension("stderr.log"), &result.stderr) + .map_err(|e| e.to_string())?; + let report: Value = serde_json::from_slice(&result.stdout) + .map_err(|e| format!("Cargo policy did not emit strict JSON: {e}"))?; + if !report_path.exists() { + save_json(report_path, &report)?; + } + if !result.status.success() { + return Err(format!( + "Cargo policy failed: {}", + report["error"].as_str().unwrap_or("see raw report") + )); + } + Ok(report) +} + +fn fixture_manifest(explicit: Option<&Path>, current: &Path) -> Result { + // Qualification binaries travel from a hosted build machine to /lab on the + // GPU runner. Source locations must be resolved from the execution host. + let selected = if let Some(path) = explicit { + current.join(path) + } else { + let local = current.join("fixtures/full-rust-gpu/Cargo.toml"); + if local.is_file() { + local + } else { + current.join("experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.toml") + } + }; + if !selected.is_file() { + return Err( + "Fixture manifest is missing; supply --fixture-manifest from the execution host".into(), + ); + } + fs::canonicalize(selected).map_err(|e| e.to_string()) +} + +fn verify(options: &Options, run: &Path, summary: &mut Value) -> Result<(), String> { + let manifest = fixture_manifest( + options.fixture.as_deref(), + &env::current_dir().map_err(|e| e.to_string())?, + )?; + let fresh_path = run.join("fresh-cargo.json"); + let cached_path = run.join("cached-cargo.json"); + summary["fixture_manifest"] = json!(manifest); + summary["fresh_cargo_report"] = json!(fresh_path); + summary["cached_cargo_report"] = json!(cached_path); + summary["owned_target_root"] = json!(run.join("targets")); + let fresh = cargo_build(options, &manifest, &run.join("targets"), &fresh_path)?; + validate_fresh_report(&fresh)?; + let (gpu, cpu) = verify_raw_receipts(&fresh, &options.backend)?; + summary["fresh_gpu_execution_verified"] = true.into(); + summary["bitcode_only_frontend_verified"] = true.into(); + summary["saved_link_without_recompilation_verified"] = true.into(); + summary["physical_gpu_units"] = gpu.into(); + summary["attributed_cpu_units"] = cpu.into(); + summary["gpu_devices"] = fresh["object_backend_receipts"]["gpu_devices"].clone(); + summary["artifact"] = fresh["artifact"].clone(); + summary["fresh_cargo_ms"] = fresh["cargo_elapsed_ms"].clone(); + let artifact = Path::new( + fresh["artifact"]["path"] + .as_str() + .ok_or("Fresh build omitted executable")?, + ); + let execution = Command::new(artifact).output().map_err(|e| e.to_string())?; + fs::write(run.join("fixture-stdout.log"), &execution.stdout).map_err(|e| e.to_string())?; + fs::write(run.join("fixture-stderr.log"), &execution.stderr).map_err(|e| e.to_string())?; + if !execution.status.success() || execution.stdout != EXPECTED.as_bytes() { + return Err(format!( + "GPU-compiled Rust fixture execution failed or produced unexpected output: {}", + String::from_utf8_lossy(&execution.stdout) + )); + } + summary["fixture_execution_verified"] = true.into(); + summary["fixture_stdout"] = EXPECTED.into(); + let cached = cargo_build(options, &manifest, &run.join("targets"), &cached_path)?; + validate_cached_report(&cached, &fresh)?; + summary["cache_only_verified"] = true.into(); + summary["cached_cargo_ms"] = cached["cargo_elapsed_ms"].clone(); + Ok(()) +} + +fn run() -> Result { + let Some(options) = options()? else { + return Ok(true); + }; + if options.report.exists() { + let prior: Value = + serde_json::from_slice(&fs::read(&options.report).map_err(|e| e.to_string())?) + .map_err(|_| "Refusing to replace a non-verification report")?; + if prior["schema"] != "gpu-rust-pipeline-verification" || prior["version"] != 1 { + return Err("Refusing to replace an unrelated report".into()); + } + } + fs::create_dir_all(&options.target_root).map_err(|e| e.to_string())?; + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| e.to_string())? + .as_nanos(); + let run = options + .target_root + .join(format!("verify-{}-{nonce}", std::process::id())); + fs::create_dir(&run).map_err(|e| format!("Could not claim verification directory: {e}"))?; + let run = fs::canonicalize(run).map_err(|e| e.to_string())?; + let mut summary = json!({"schema":"gpu-rust-pipeline-verification","version":1,"status":"running","scope":"full-rust-fixture-cargo-object-emission","backend":options.backend,"toolchain":"1.98.1","performance_speedup_demonstrated":false,"compiler_size_limit_qualified":false,"fresh_gpu_execution_verified":false,"cache_only_verified":false,"fixture_execution_verified":false,"bitcode_only_frontend_verified":false,"saved_link_without_recompilation_verified":false,"owned_verification_directory":run}); + let result = verify(&options, &run, &mut summary); + summary["status"] = if result.is_ok() { "ok" } else { "failed" }.into(); + if let Err(error) = &result { + summary["error"] = error.clone().into(); + } + save_json(&options.report, &summary)?; + println!( + "{}", + serde_json::to_string(&summary).map_err(|e| e.to_string())? + ); + Ok(result.is_ok()) +} + +fn main() { + match run() { + Ok(true) => {} + Ok(false) => std::process::exit(1), + Err(error) => { + eprintln!("gpu-verify-pipeline: {error}"); + println!( + "{}", + json!({"schema":"gpu-rust-pipeline-verification","version":1,"status":"failed","error":error}) + ); + std::process::exit(1); + } + } +} + +#[cfg(test)] +mod tests { + #[test] + fn moved_verifier_resolves_the_runtime_fixture_and_explicit_manifest() { + let root = std::env::temp_dir().join(format!("gpu-runtime-fixture-{}", std::process::id())); + std::fs::create_dir_all(root.join("fixtures/full-rust-gpu")).unwrap(); + let default = root.join("fixtures/full-rust-gpu/Cargo.toml"); + let explicit = root.join("custom-fixture.toml"); + std::fs::write(&default, "[workspace]\n").unwrap(); + std::fs::write(&explicit, "[workspace]\n").unwrap(); + assert_eq!( + super::fixture_manifest(None, &root).unwrap(), + std::fs::canonicalize(&default).unwrap() + ); + assert_eq!( + super::fixture_manifest(Some(&explicit), &root).unwrap(), + std::fs::canonicalize(&explicit).unwrap() + ); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] + fn metadata_only_or_simulated_reports_cannot_qualify_a_gpu_pipeline() { + let report = serde_json::json!({"status":"ok","gpu_executed":false,"gpu_compilation_jobs":0,"current_run_gpu_qualified":false,"performance_speedup_demonstrated":false,"artifact":{"cargo_fresh":true},"object_backend_receipts":{"physical_gpu_units":0}}); + assert!(super::validate_fresh_report(&report).is_err()); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/bitcode_linker.rs b/experiments/gpu-rust-compiler/compiler/src/bitcode_linker.rs new file mode 100644 index 00000000000..18535997a82 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/bitcode_linker.rs @@ -0,0 +1,1103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Experimental RUSTC_WRAPPER: Rust emits bitcode objects, the selected adapter +//! emits native objects, and Rust links/packages its own saved metadata. Native +//! Rust object emission does not run before the adapter. This serial prototype +//! has no object cache, Cargo-wide service, or extra jobserver credit acquisition. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{ + env, + ffi::OsString, + fs, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::atomic::{AtomicU64, Ordering}, + time::Instant, +}; + +const RUST_VERSION: &str = "1.98.1"; +static NEXT: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug)] +pub struct CompilePlan { + pub llvm_codegen_opt_level: u32, + pub source: PathBuf, + pub source_argument: usize, + pub output_directory: PathBuf, + out_dir: Option<(usize, bool)>, + output: Option<(usize, bool, OsString)>, +} +#[derive(Clone, Debug)] +pub enum InvocationKind { + Compile(CompilePlan), + PassThrough(String), +} + +pub fn classify(arguments: &[OsString]) -> Result { + let pass = |reason: &str| Ok(InvocationKind::PassThrough(reason.into())); + let Some(text): Option> = arguments.iter().map(|s| s.to_str()).collect() else { + return pass("non-UTF-8 arguments use original rustc"); + }; + if text.iter().any(|s| { + s.starts_with('@') + || *s == "-" + || s.starts_with("--print") + || *s == "--version" + || *s == "-vV" + || *s == "--help" + || *s == "-h" + }) { + return pass("query, response file, or stdin uses original rustc"); + } + if text + .iter() + .any(|s| s.contains("no-link") || s.contains("link-only") || s.contains("save-temps")) + { + return pass("existing split-link or saved-temporary invocation uses original rustc"); + } + let mut source = None; + let mut out_dir = None; + let mut output = None; + let mut has_link = true; + let mut llvm_codegen_opt_level = 0; + let mut index = 0; + while index < text.len() { + let value = text[index]; + let optimization = if value == "-O" { + Some("2") + } else if value == "-C" { + text.get(index + 1) + .and_then(|s| s.strip_prefix("opt-level=")) + } else { + value.strip_prefix("-Copt-level=") + }; + if let Some(level) = optimization { + // Match pinned rustc's target-machine level mapping. Size modes + // carry their size attributes in IR and use CodeGenOpt::Default. + llvm_codegen_opt_level = match level { + "0" => 0, + "1" => 1, + "2" | "s" | "z" => 2, + "3" => 3, + _ => return pass("unsupported optimization level uses original rustc"), + }; + } + let paired = matches!( + value, + "--emit" + | "--crate-type" + | "--crate-name" + | "--out-dir" + | "-o" + | "--edition" + | "--target" + | "--error-format" + | "--json" + | "--extern" + | "--cfg" + | "--check-cfg" + | "--cap-lints" + | "--remap-path-prefix" + | "--sysroot" + | "--color" + | "--diagnostic-width" + | "-C" + | "-Z" + | "-L" + | "-A" + | "-W" + | "-D" + | "-F" + ); + let mut emit = value.strip_prefix("--emit="); + if value == "--emit" { + emit = text.get(index + 1).copied(); + } + if let Some(emit) = emit { + if emit.contains('=') + || emit + .split(',') + .any(|kind| !matches!(kind, "dep-info" | "metadata" | "link")) + { + return pass("explicit or additional emit destinations use original rustc"); + } + has_link = emit.split(',').any(|kind| kind == "link"); + } + if value == "--out-dir" { + let next = arguments.get(index + 1).ok_or("--out-dir has no value")?; + if out_dir.is_some() { + return pass("multiple output directories use original rustc"); + } + out_dir = Some((index + 1, false, PathBuf::from(next))); + } else if let Some(path) = value.strip_prefix("--out-dir=") { + if out_dir.is_some() { + return pass("multiple output directories use original rustc"); + } + out_dir = Some((index, true, PathBuf::from(path))); + } else if value == "-o" { + let next = arguments.get(index + 1).ok_or("-o has no value")?; + if output.is_some() { + return pass("multiple output files use original rustc"); + } + output = Some((index + 1, false, PathBuf::from(next))); + } else if let Some(path) = value.strip_prefix("-o=") { + if output.is_some() { + return pass("multiple output files use original rustc"); + } + output = Some((index, true, PathBuf::from(path))); + } else if !value.starts_with('-') { + if !value.ends_with(".rs") || source.is_some() { + return pass("unrecognized source invocation uses original rustc"); + } + source = Some((index, PathBuf::from(value))); + } + // Split debug artifacts need a separately qualified publication path. + if value.starts_with("-Cdebuginfo=") && value != "-Cdebuginfo=0" { + return pass("debug artifacts use original rustc until qualified"); + } + if value == "-C" + && text + .get(index + 1) + .is_some_and(|s| s.starts_with("debuginfo=") && *s != "debuginfo=0") + { + return pass("debug artifacts use original rustc until qualified"); + } + index += if paired { 2 } else { 1 }; + } + if !has_link { + return pass("metadata-only invocation performs no object emission"); + } + let Some((source_argument, source)) = source else { + return pass("no supported source file"); + }; + let output_directory = if let Some((_, _, path)) = &output { + path.parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")) + .to_owned() + } else if let Some((_, _, path)) = &out_dir { + path.clone() + } else { + return pass("an explicit output directory or file is required"); + }; + // These paths become Make dep-info targets; complex escaping is passed + // through rather than rewriting a dependency contract incorrectly. + if output_directory + .to_string_lossy() + .chars() + .any(|c| matches!(c, ' ' | '\n' | '\r' | '#' | '$' | '\\')) + { + return pass("output path needs unsupported dep-info escaping"); + } + let output = output + .map(|(index, inline, path)| -> Result<_, String> { + let name = path + .file_name() + .ok_or("output file has no basename")? + .to_owned(); + Ok((index, inline, name)) + }) + .transpose()?; + Ok(InvocationKind::Compile(CompilePlan { + llvm_codegen_opt_level, + source, + source_argument, + output_directory, + out_dir: out_dir.map(|(i, inline, _)| (i, inline)), + output, + })) +} + +#[derive(Clone, Debug)] +pub struct WrapperOptions { + pub controller: PathBuf, + pub bridge: PathBuf, + pub llc: PathBuf, + pub backend: String, + pub allow_fallback: bool, + pub receipt_directory: PathBuf, + pub quality: String, + pub gpu: Option, + pub capture_directory: Option, +} +#[derive(Clone, Debug)] +pub struct GpuOptions { + pub emitter: PathBuf, + pub backend: String, + pub library: PathBuf, + pub shader: Option, + pub device: u32, +} +impl WrapperOptions { + pub fn from_env() -> Result { + let path = |name: &str| { + env::var_os(name) + .filter(|s| !s.is_empty()) + .map(PathBuf::from) + .ok_or_else(|| format!("{name} must name an executable or receipt directory")) + }; + let backend = env::var("GPU_LINK_BACKEND").unwrap_or_else(|_| "tpde".into()); + if !matches!(backend.as_str(), "tpde" | "llvm" | "gpu") { + return Err("GPU_LINK_BACKEND must be tpde, llvm, or gpu".into()); + } + let allow_fallback = match env::var("GPU_LINK_ALLOW_FALLBACK") + .as_deref() + .unwrap_or("0") + { + "0" => false, + "1" => true, + _ => return Err("GPU_LINK_ALLOW_FALLBACK must be 0 or 1".into()), + }; + let gpu = if backend == "gpu" { + let gpu_backend = env::var("GPU_LINK_GPU_BACKEND") + .map_err(|_| "GPU_LINK_GPU_BACKEND must be metal or cuda")?; + if !matches!(gpu_backend.as_str(), "metal" | "cuda") { + return Err("GPU_LINK_GPU_BACKEND must be metal or cuda".into()); + } + Some(GpuOptions { + emitter: path("GPU_LINK_GPU_EMITTER")?, + backend: gpu_backend, + library: path("GPU_LINK_GPU_LIBRARY")?, + shader: env::var_os("GPU_LINK_GPU_SHADER") + .filter(|s| !s.is_empty()) + .map(PathBuf::from), + device: env::var("GPU_LINK_GPU_DEVICE") + .unwrap_or_else(|_| "0".into()) + .parse() + .map_err(|_| "GPU_LINK_GPU_DEVICE must be a u32 device ordinal")?, + }) + } else { + None + }; + Ok(Self { + controller: path("GPU_LINK_CONTROLLER")?, + bridge: path("GPU_LINK_BRIDGE")?, + llc: path("GPU_LINK_LLC")?, + backend, + allow_fallback, + receipt_directory: path("GPU_LINK_RECEIPTS")?, + quality: env::var("GPU_LINK_QUALITY").unwrap_or_else(|_| "unspecified".into()), + gpu, + capture_directory: env::var_os("GPU_LINK_CAPTURE_DIR") + .map(|path| { + if path.is_empty() { + Err(String::from( + "GPU_LINK_CAPTURE_DIR must be nonempty when set", + )) + } else { + Ok(PathBuf::from(path)) + } + }) + .transpose()?, + }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct UnitReport { + pub input_kind: String, + pub input_sha256: String, + pub object_sha256: String, + pub backend: String, + pub target: String, + pub fallback_reason: Option, + pub controller_elapsed_ms: f64, + pub controller_receipt: Value, + pub physical_gpu: bool, + pub device_id: Option, + pub kernel_dispatches: u64, + pub kernel_output_used: bool, +} + +#[derive(Debug, Serialize)] +struct CapturedUnit { + unit_index: usize, + original_filename: String, + captured_filename: String, + input_sha256: String, + input_bytes: u64, + llvm_codegen_opt_level: u32, + target: Option, + controller_elapsed_ms: Option, + emission_receipt: Option, +} + +#[derive(Debug, Serialize)] +struct CaptureManifest { + schema: &'static str, + version: u32, + scope: &'static str, + status: String, + rustc_version_verbose: String, + source: PathBuf, + input_arguments_sha256: String, + requested_backend: String, + quality: String, + llvm_codegen_opt_level: u32, + units: Vec, +} +#[derive(Debug, Serialize, Deserialize)] +pub struct InvocationReport { + pub schema: String, + pub version: u32, + pub status: String, + pub rustc_version_verbose: String, + pub requested_backend: String, + pub quality: String, + pub gpu_executed: bool, + pub performance_speedup_demonstrated: bool, + pub frontend_bitcode_verified: bool, + pub native_object_before_selected_emitter: Option, + pub saved_link_without_recompilation: bool, + pub forwarded_passthrough: bool, + pub passthrough_reason: Option, + pub input_arguments_sha256: String, + pub source: Option, + pub staging_directory: Option, + pub capture_manifest: Option, + pub frontend_ms: f64, + pub materialize_ms: f64, + pub emit_ms: f64, + pub link_ms: f64, + pub total_ms: f64, + pub units: Vec, + pub exit_code: i32, + pub error: Option, +} +pub struct WrapperOutcome { + pub report: InvocationReport, + pub exit_code: i32, +} + +#[derive(Clone, Debug)] +pub struct DiagnosticRelay { + stage: PathBuf, + destination: PathBuf, +} +impl DiagnosticRelay { + fn publish(&self, path: &Path) -> Result<(), String> { + let relative = path + .strip_prefix(&self.stage) + .map_err(|_| "compiler artifact is outside owned staging")?; + if relative.components().count() != 1 { + return Err("nested compiler artifact is not qualified".into()); + } + let name = relative.to_str().ok_or("artifact path must be UTF-8")?; + let publish = self.stage.join(".publish"); + fs::create_dir_all(&publish).map_err(|e| e.to_string())?; + let candidate = publish.join(name); + if candidate.exists() { + fs::remove_file(&candidate).map_err(|e| e.to_string())?; + } + if name.ends_with(".d") { + let content = fs::read_to_string(path).map_err(|e| e.to_string())?; + fs::write( + &candidate, + content.replace( + self.stage.to_str().ok_or("staging path must be UTF-8")?, + self.destination + .to_str() + .ok_or("output path must be UTF-8")?, + ), + ) + .map_err(|e| e.to_string())?; + } else { + fs::hard_link(path, &candidate).map_err(|e| e.to_string())?; + } + fs::rename(candidate, self.destination.join(relative)) + .map_err(|e| format!("publish compiler artifact: {e}")) + } + fn forward(&self, bytes: &[u8]) -> Result<(), String> { + let mut transformed = None; + if let Ok(mut message) = serde_json::from_slice::(bytes) { + if message.get("$message_type").and_then(Value::as_str) == Some("artifact") { + if let Some(name) = message.get("artifact").and_then(Value::as_str) { + let path = PathBuf::from(name); + if path.starts_with(&self.stage) { + // Cargo can start dependent compilation as soon as this + // metadata event is forwarded; publish that file first. + if path.extension().is_some_and(|e| e == "rmeta") { + self.publish(&path)?; + } + let relative = path.strip_prefix(&self.stage).unwrap(); + message["artifact"] = Value::String( + self.destination + .join(relative) + .to_string_lossy() + .into_owned(), + ); + let mut output = serde_json::to_vec(&message).map_err(|e| e.to_string())?; + output.push(b'\n'); + transformed = Some(output); + } + } + } + } + std::io::stderr() + .lock() + .write_all(transformed.as_deref().unwrap_or(bytes)) + .map_err(|e| e.to_string()) + } +} + +#[derive(Clone, Debug)] +pub struct ToolCall { + pub program: PathBuf, + pub arguments: Vec, + pub bootstrap: bool, + pub inherit_io: bool, + pub relay: Option, +} +pub struct ToolOutput { + pub exit_code: i32, + pub stdout: Vec, + pub stderr: Vec, +} +pub trait ToolRunner { + fn run(&mut self, call: &ToolCall) -> Result; +} +pub struct ProcessRunner; +fn exit_code(status: std::process::ExitStatus) -> i32 { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + status + .code() + .unwrap_or_else(|| 128 + status.signal().unwrap_or(1)) + } + #[cfg(not(unix))] + { + status.code().unwrap_or(1) + } +} +impl ToolRunner for ProcessRunner { + fn run(&mut self, call: &ToolCall) -> Result { + let mut command = Command::new(&call.program); + command.args(&call.arguments); + if call.bootstrap { + command.env("RUSTC_BOOTSTRAP", "1"); + } + if call.inherit_io { + let status = command.status().map_err(|e| e.to_string())?; + return Ok(ToolOutput { + exit_code: exit_code(status), + stdout: vec![], + stderr: vec![], + }); + } + if let Some(relay) = &call.relay { + let mut child = command + .stdout(Stdio::inherit()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| e.to_string())?; + let stderr = child.stderr.take().unwrap(); + let mut reader = BufReader::new(stderr); + let mut line = Vec::new(); + let mut failure = None; + loop { + line.clear(); + match reader.read_until(b'\n', &mut line) { + Ok(0) => break, + Ok(_) => { + if let Err(error) = relay.forward(&line) { + failure.get_or_insert(error); + } + } + Err(error) => { + failure = Some(error.to_string()); + break; + } + } + } + let status = child.wait().map_err(|e| e.to_string())?; + if let Some(error) = failure { + return Err(format!("compiler diagnostic relay: {error}")); + } + return Ok(ToolOutput { + exit_code: exit_code(status), + stdout: vec![], + stderr: vec![], + }); + } + let output = command.output().map_err(|e| e.to_string())?; + Ok(ToolOutput { + exit_code: exit_code(output.status), + stdout: output.stdout, + stderr: output.stderr, + }) + } +} + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +fn argument_digest(arguments: &[OsString]) -> String { + let mut bytes = Vec::new(); + for argument in arguments { + let text = argument.to_string_lossy(); + bytes.extend_from_slice(&(text.len() as u64).to_le_bytes()); + bytes.extend_from_slice(text.as_bytes()); + } + digest(&bytes) +} +fn call(program: &Path, arguments: Vec) -> ToolCall { + ToolCall { + program: program.to_owned(), + arguments, + bootstrap: false, + inherit_io: false, + relay: None, + } +} +fn elapsed(start: Instant) -> f64 { + start.elapsed().as_secs_f64() * 1000.0 +} +fn absolute(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_owned()) + } else { + Ok(env::current_dir().map_err(|e| e.to_string())?.join(path)) + } +} + +fn emit_object( + options: &WrapperOptions, + llvm_codegen_opt_level: u32, + input: &Path, + output: &Path, + receipt: &Path, + runner: &mut impl ToolRunner, +) -> Result<(Value, Option, bool), String> { + let mut gpu_fallback = None; + if let Some(gpu) = &options.gpu { + let packet = output.with_extension("gem1"); + let attempt = (|| -> Result { + let export = runner.run(&call( + &options.bridge, + vec![ + "--export-leaf".into(), + input.as_os_str().to_owned(), + packet.clone().into_os_string(), + ], + ))?; + if export.exit_code != 0 { + return Err(format!( + "GPU scalar eligibility/export rejected module: {}", + String::from_utf8_lossy(&export.stderr).trim() + )); + } + let mut arguments = vec![ + "--input".into(), + packet.into_os_string(), + "--output".into(), + output.as_os_str().to_owned(), + "--backend".into(), + gpu.backend.clone().into(), + "--library".into(), + gpu.library.clone().into_os_string(), + "--device".into(), + gpu.device.to_string().into(), + "--report".into(), + receipt.as_os_str().to_owned(), + ]; + if let Some(shader) = &gpu.shader { + arguments.extend(["--shader".into(), shader.clone().into_os_string()]); + } + let emission = runner.run(&call(&gpu.emitter, arguments))?; + if !emission.stderr.is_empty() { + std::io::stderr() + .lock() + .write_all(&emission.stderr) + .map_err(|e| e.to_string())?; + } + if emission.exit_code != 0 { + return Err(format!( + "physical GPU emission failed with exit code {}", + emission.exit_code + )); + } + let value: Value = + serde_json::from_slice(&fs::read(receipt).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if value.get("backend").and_then(Value::as_str) != Some(&gpu.backend) + || value.get("physical_gpu").and_then(Value::as_bool) != Some(true) + || value.get("gpu_executed").and_then(Value::as_bool) != Some(true) + || value + .get("device_id") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + || value + .get("kernel_dispatches") + .and_then(Value::as_u64) + .unwrap_or(0) + == 0 + || value.get("kernel_output_used").and_then(Value::as_bool) != Some(true) + { + return Err("GPU emitter did not provide matching physical-device and used-kernel-output evidence".into()); + } + Ok(value) + })(); + match attempt { + Ok(value) => return Ok((value, None, true)), + Err(error) if options.allow_fallback => { + // Failed candidates are never linked. The immutable bitcode is + // replayed by an explicitly permitted CPU adapter, with cost. + if output.exists() { + fs::remove_file(output).map_err(|e| e.to_string())?; + } + if receipt.exists() { + fs::remove_file(receipt).map_err(|e| e.to_string())?; + } + gpu_fallback = Some(error); + } + Err(error) => return Err(error), + } + } + let backend = if options.backend == "gpu" { + "tpde" + } else { + &options.backend + }; + let mut args = vec![ + "compile".into(), + "--input".into(), + input.as_os_str().to_owned(), + "--output".into(), + output.as_os_str().to_owned(), + "--bridge".into(), + options.bridge.clone().into_os_string(), + "--llc".into(), + options.llc.clone().into_os_string(), + "--backend".into(), + backend.into(), + "--llvm-codegen-opt-level".into(), + llvm_codegen_opt_level.to_string().into(), + "--report".into(), + receipt.as_os_str().to_owned(), + ]; + if options.allow_fallback { + args.push("--allow-fallback".into()); + } + let result = runner.run(&call(&options.controller, args))?; + if !result.stderr.is_empty() { + std::io::stderr() + .lock() + .write_all(&result.stderr) + .map_err(|e| e.to_string())?; + } + if result.exit_code != 0 { + return Err(format!( + "object adapter failed with exit code {}", + result.exit_code + )); + } + let value: Value = serde_json::from_slice(&fs::read(receipt).map_err(|e| e.to_string())?) + .map_err(|e| format!("invalid object-controller receipt: {e}"))?; + if !matches!( + value.get("backend").and_then(Value::as_str), + Some("tpde" | "llvm") + ) || value.get("gpu_accelerated").and_then(Value::as_bool) != Some(false) + { + return Err("CPU object controller cannot report GPU execution".into()); + } + Ok((value, gpu_fallback, false)) +} + +pub fn run_with( + rustc: &Path, + arguments: &[OsString], + options: &WrapperOptions, + runner: &mut impl ToolRunner, +) -> Result { + let start = Instant::now(); + let number = NEXT.fetch_add(1, Ordering::Relaxed); + let identifier = format!("gpu-object-bridge-{}-{number}", std::process::id()); + let mut capture: Option = None; + let mut report = InvocationReport { + schema: "gpu-rust-object-bridge".into(), + version: 1, + status: "failed".into(), + rustc_version_verbose: String::new(), + requested_backend: options.backend.clone(), + quality: options.quality.clone(), + gpu_executed: false, + performance_speedup_demonstrated: false, + frontend_bitcode_verified: false, + native_object_before_selected_emitter: None, + saved_link_without_recompilation: false, + forwarded_passthrough: false, + passthrough_reason: None, + input_arguments_sha256: argument_digest(arguments), + source: None, + staging_directory: None, + capture_manifest: None, + frontend_ms: 0.0, + materialize_ms: 0.0, + emit_ms: 0.0, + link_ms: 0.0, + total_ms: 0.0, + units: vec![], + exit_code: 1, + error: None, + }; + let operation = (|| -> Result { + let plan = match classify(arguments)? { + InvocationKind::PassThrough(reason) => { + report.forwarded_passthrough = true; + report.passthrough_reason = Some(reason); + let mut invocation = call(rustc, arguments.to_vec()); + invocation.inherit_io = true; + return runner.run(&invocation).map(|output| output.exit_code); + } + InvocationKind::Compile(plan) => plan, + }; + report.source = Some(absolute(&plan.source)?); + if options.backend == "gpu" && options.gpu.is_none() { + return Err("GPU object mode requires explicit device/emitter configuration".into()); + } + let materialize = Instant::now(); + let version = runner.run(&call(rustc, vec!["-vV".into()]))?; + report.rustc_version_verbose = + String::from_utf8(version.stdout).map_err(|_| "rustc version is not UTF-8")?; + if version.exit_code != 0 + || !report + .rustc_version_verbose + .lines() + .any(|line| line == format!("release: {RUST_VERSION}")) + { + return Err(format!("object bridge requires exact rustc {RUST_VERSION}")); + } + let destination = absolute(&plan.output_directory)?; + fs::create_dir_all(&destination).map_err(|e| e.to_string())?; + let stage = destination.join(format!(".{identifier}")); + fs::create_dir(&stage) + .map_err(|e| format!("create exclusively owned compiler staging: {e}"))?; + report.staging_directory = Some(stage.clone()); + if let Some(root) = &options.capture_directory { + let root = absolute(root)?; + fs::create_dir_all(&root) + .map_err(|error| format!("Create audit capture parent: {error}"))?; + let directory = root.join(&identifier); + fs::create_dir(&directory) + .map_err(|error| format!("Claim exclusive audit capture directory: {error}"))?; + report.capture_manifest = Some(directory.join("manifest.json")); + capture = Some(CaptureManifest { + schema: "gpu-rust-bitcode-capture", + version: 1, + scope: "audit-only exact frontend bitcode corpus", + status: "failed".into(), + rustc_version_verbose: report.rustc_version_verbose.clone(), + source: report.source.clone().ok_or("Capture has no source")?, + input_arguments_sha256: report.input_arguments_sha256.clone(), + requested_backend: options.backend.clone(), + quality: options.quality.clone(), + llvm_codegen_opt_level: plan.llvm_codegen_opt_level, + units: Vec::new(), + }); + } + let relay = DiagnosticRelay { + stage: stage.clone(), + destination, + }; + let mut frontend = arguments.to_vec(); + if let Some((index, inline)) = plan.out_dir { + frontend[index] = if inline { + format!("--out-dir={}", stage.display()).into() + } else { + stage.clone().into_os_string() + }; + } + if let Some((index, inline, name)) = &plan.output { + let path = stage.join(name); + frontend[*index] = if *inline { + format!("-o={}", path.display()).into() + } else { + path.into_os_string() + }; + } + frontend.extend([ + "-Clinker-plugin-lto=yes".into(), + "-Clto=off".into(), + "-Zno-link".into(), + "-Zallow-features=".into(), + ]); + report.materialize_ms += elapsed(materialize); + let mut invocation = call(rustc, frontend); + invocation.bootstrap = true; + invocation.relay = Some(relay.clone()); + let time = Instant::now(); + let compiled = runner.run(&invocation)?; + report.frontend_ms = elapsed(time); + if compiled.exit_code != 0 { + return Ok(compiled.exit_code); + } + let time = Instant::now(); + let mut objects = vec![]; + let mut rlinks = vec![]; + for entry in fs::read_dir(&stage).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if !entry.file_type().map_err(|e| e.to_string())?.is_file() { + continue; + } + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.ends_with(".rcgu.o") { + objects.push(path); + } else if name.ends_with(".rlink") { + rlinks.push(path); + } + } + objects.sort(); + if objects.is_empty() || rlinks.len() != 1 { + return Err( + "owned Rust staging must contain bitcode objects and exactly one saved-link record" + .into(), + ); + } + let backend_stage = stage.join(".backend"); + fs::create_dir(&backend_stage).map_err(|e| e.to_string())?; + report.materialize_ms += elapsed(time); + for (index, object) in objects.iter().enumerate() { + let time = Instant::now(); + let bytes = fs::read(object).map_err(|e| e.to_string())?; + if !bytes.starts_with(b"BC\xc0\xde") && !bytes.starts_with(b"\xde\xc0\x17\x0b") { + report.native_object_before_selected_emitter = + object::File::parse(bytes.as_slice()).ok().map(|_| true); + return Err( + "rustc produced a native object before the selected object adapter".into(), + ); + } + let input_sha256 = digest(&bytes); + if let Some(capture) = capture.as_mut() { + let captured_filename = format!("unit-{index}.bc"); + let directory = report + .capture_manifest + .as_ref() + .and_then(|path| path.parent()) + .ok_or("Audit capture has no owned directory")?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join(&captured_filename)) + .map_err(|error| format!("Create immutable bitcode capture: {error}"))?; + file.write_all(&bytes) + .map_err(|error| format!("Write immutable bitcode capture: {error}"))?; + capture.units.push(CapturedUnit { + unit_index: index, + original_filename: object + .file_name() + .and_then(|name| name.to_str()) + .ok_or("Captured input name is not UTF-8")? + .into(), + captured_filename, + input_sha256: input_sha256.clone(), + input_bytes: bytes.len() as u64, + llvm_codegen_opt_level: plan.llvm_codegen_opt_level, + target: None, + controller_elapsed_ms: None, + emission_receipt: None, + }); + } + let converted = backend_stage.join(format!("{index}.o")); + let receipt = backend_stage.join(format!("{index}.json")); + report.materialize_ms += elapsed(time); + let time = Instant::now(); + let emission = emit_object( + options, + plan.llvm_codegen_opt_level, + object, + &converted, + &receipt, + runner, + ); + let controller_elapsed_ms = elapsed(time); + report.emit_ms += controller_elapsed_ms; + let (value, gpu_fallback, physical_gpu) = emission?; + let time = Instant::now(); + let backend = value + .get("backend") + .and_then(Value::as_str) + .filter(|b| matches!(*b, "tpde" | "llvm" | "metal" | "cuda")) + .ok_or("controller receipt has no qualified backend")? + .to_owned(); + let target = value + .get("target") + .and_then(Value::as_str) + .ok_or("controller receipt has no target")? + .to_owned(); + if physical_gpu != matches!(backend.as_str(), "metal" | "cuda") { + return Err("object backend contradicts physical execution attribution".into()); + } + if !physical_gpu && value.get("gpu_accelerated").and_then(Value::as_bool) != Some(false) + { + return Err("CPU object controller cannot report GPU execution".into()); + } + if options.backend == "llvm" && backend != "llvm" { + return Err("LLVM request used a different object backend".into()); + } + let converted_bytes = fs::read(&converted).map_err(|e| e.to_string())?; + object::File::parse(converted_bytes.as_slice()) + .map_err(|e| format!("converted object is invalid: {e}"))?; + if value.get("object_bytes").and_then(Value::as_u64) + != Some(converted_bytes.len() as u64) + { + return Err("controller object byte count differs from emitted object".into()); + } + let cpu_fallback_reason = value + .get("fallback_reason") + .and_then(Value::as_str) + .map(str::to_owned); + let fallback_reason = match (gpu_fallback, cpu_fallback_reason) { + (Some(gpu), Some(cpu)) => Some(format!("{gpu}; {cpu}")), + (Some(reason), None) | (None, Some(reason)) => Some(reason), + (None, None) => None, + }; + if options.backend == "tpde" + && backend != "tpde" + && (!options.allow_fallback || fallback_reason.is_none()) + { + return Err("controller used unqualified implicit LLVM fallback".into()); + } + report.units.push(UnitReport { + input_kind: "llvm-bitcode".into(), + input_sha256, + object_sha256: digest(&converted_bytes), + backend, + target, + fallback_reason, + controller_elapsed_ms, + physical_gpu, + device_id: value + .get("device_id") + .and_then(Value::as_str) + .map(str::to_owned), + kernel_dispatches: value + .get("kernel_dispatches") + .and_then(Value::as_u64) + .unwrap_or(0), + kernel_output_used: value + .get("kernel_output_used") + .and_then(Value::as_bool) + .unwrap_or(false), + controller_receipt: value, + }); + report.gpu_executed |= physical_gpu; + if let Some(capture) = capture.as_mut() { + let emitted = report + .units + .last() + .ok_or("Captured module has no emission receipt")?; + let captured = capture + .units + .last_mut() + .ok_or("Captured module was not saved")?; + captured.target = Some(emitted.target.clone()); + captured.controller_elapsed_ms = Some(controller_elapsed_ms); + captured.emission_receipt = Some(emitted.clone()); + } + // Replace only an object in this invocation's private directory. + fs::rename(converted, object).map_err(|e| e.to_string())?; + report.materialize_ms += elapsed(time); + } + report.frontend_bitcode_verified = true; + report.native_object_before_selected_emitter = Some(false); + let mut link = arguments.to_vec(); + link[plan.source_argument] = rlinks[0].clone().into_os_string(); + if let Some((index, inline)) = plan.out_dir { + link[index] = if inline { + format!("--out-dir={}", stage.display()).into() + } else { + stage.clone().into_os_string() + }; + } + if let Some((index, inline, name)) = &plan.output { + let path = stage.join(name); + link[*index] = if *inline { + format!("-o={}", path.display()).into() + } else { + path.into_os_string() + }; + } + link.push("-Zlink-only".into()); + let mut invocation = call(rustc, link); + invocation.bootstrap = true; + invocation.relay = Some(relay.clone()); + let time = Instant::now(); + let linked = runner.run(&invocation)?; + report.link_ms = elapsed(time); + if linked.exit_code != 0 { + return Ok(linked.exit_code); + } + report.saved_link_without_recompilation = true; + let time = Instant::now(); + for entry in fs::read_dir(&stage).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if !entry.file_type().map_err(|e| e.to_string())?.is_file() { + continue; + } + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.ends_with(".rlink") || name.ends_with(".rcgu.o") { + continue; + } + relay.publish(&entry.path())?; + } + fs::remove_dir_all(&stage).map_err(|e| format!("clean owned staging: {e}"))?; + report.materialize_ms += elapsed(time); + Ok(0) + })(); + report.exit_code = match operation { + Ok(code) => code, + Err(error) => { + report.error = Some(error); + 1 + } + }; + report.status = if report.exit_code != 0 { + "failed" + } else if report.forwarded_passthrough { + "passthrough" + } else { + "success" + } + .into(); + if let Some(mut capture) = capture { + let time = Instant::now(); + capture.status = report.status.clone(); + let path = report + .capture_manifest + .as_ref() + .ok_or("Capture manifest path is missing")?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|error| format!("Create final audit capture manifest: {error}"))?; + serde_json::to_writer(&mut file, &capture) + .map_err(|error| format!("Write final audit capture manifest: {error}"))?; + report.materialize_ms += elapsed(time); + } + report.total_ms = elapsed(start); + fs::create_dir_all(&options.receipt_directory).map_err(|e| e.to_string())?; + let path = options.receipt_directory.join(format!("{identifier}.json")); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| format!("create invocation receipt: {e}"))?; + serde_json::to_writer(&mut file, &report).map_err(|e| e.to_string())?; + Ok(WrapperOutcome { + exit_code: report.exit_code, + report, + }) +} diff --git a/experiments/gpu-rust-compiler/compiler/src/cargo_policy.rs b/experiments/gpu-rust-compiler/compiler/src/cargo_policy.rs new file mode 100644 index 00000000000..a5e314503ee --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/cargo_policy.rs @@ -0,0 +1,1819 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Compatible Cargo policies. This module performs no GPU compilation. + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + env, fs, + io::{BufRead, BufReader, Read, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +pub const TOOLCHAIN: &str = "1.98.1"; +const BACKEND: &str = "rustc-llvm-cpu"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ObjectBackend { + Native, + Tpde, + LlvmBitcode, + Gpu, +} + +impl ObjectBackend { + pub fn parse(value: &str) -> Result { + match value { + "native" => Ok(Self::Native), + "tpde" => Ok(Self::Tpde), + "llvm-bitcode" => Ok(Self::LlvmBitcode), + "gpu" => Ok(Self::Gpu), + _ => Err("Object backend must be native, tpde, llvm-bitcode, or gpu".into()), + } + } + fn name(self) -> &'static str { + match self { + Self::Native => "native", + Self::Tpde => "tpde", + Self::LlvmBitcode => "llvm-bitcode", + Self::Gpu => "gpu", + } + } + fn controller_name(self) -> &'static str { + match self { + Self::Tpde => "tpde", + Self::Gpu => "gpu", + _ => "llvm", + } + } + fn planned(self) -> &'static str { + match self { + Self::Native => BACKEND, + Self::Tpde => "rustc-bitcode-tpde-cpu", + Self::LlvmBitcode => "rustc-bitcode-llvm-cpu", + Self::Gpu => "rustc-bitcode-gpu-eligible", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Quality { + Fast, + Balanced, + Release, +} + +impl Quality { + pub fn parse(value: &str) -> Result { + match value { + "fast" => Ok(Self::Fast), + "balanced" => Ok(Self::Balanced), + "release" => Ok(Self::Release), + _ => Err("Quality must be fast, balanced, or release".into()), + } + } + + pub fn name(self) -> &'static str { + match self { + Self::Fast => "fast", + Self::Balanced => "balanced", + Self::Release => "release", + } + } + + fn config(self) -> Vec { + let opt_level = match self { + Self::Fast => 1, + Self::Balanced => 2, + Self::Release => return Vec::new(), + }; + vec![ + format!("profile.release.opt-level={opt_level}"), + "profile.release.codegen-units=16".into(), + "profile.release.lto=\"off\"".into(), + ] + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Placement { + Auto, + Cpu, + Metal, + Cuda, +} + +impl Placement { + pub fn parse(value: &str) -> Result { + match value { + "auto" => Ok(Self::Auto), + "cpu" => Ok(Self::Cpu), + "metal" => Ok(Self::Metal), + "cuda" => Ok(Self::Cuda), + _ => Err("Placement must be auto, cpu, metal, or cuda".into()), + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum SizeBasis { + Unmodified, + StripDebug, +} + +impl SizeBasis { + pub fn parse(value: &str) -> Result { + match value { + "unmodified" => Ok(Self::Unmodified), + "strip-debug" => Ok(Self::StripDebug), + _ => Err("Size basis must be unmodified or strip-debug".into()), + } + } +} + +pub struct Options { + pub plan: bool, + pub manifest_path: PathBuf, + pub package: String, + pub binary: String, + pub quality: Quality, + pub placement: Placement, + pub target_root: Option, + pub offline: bool, + pub features: Option, + pub no_default_features: bool, + pub reference_artifact: Option, + pub max_size_ratio: f64, + pub size_basis: SizeBasis, + pub object_backend: ObjectBackend, + pub bitcode_wrapper: Option, + pub tpde_controller: Option, + pub tpde_bridge: Option, + pub llvm_llc: Option, + pub allow_backend_fallback: bool, + pub gpu_emitter: Option, + pub gpu_library: Option, + pub gpu_shader: Option, + pub gpu_backend: Option, + pub gpu_device: u32, +} + +impl Default for Options { + fn default() -> Self { + Self { + plan: false, + manifest_path: "Cargo.toml".into(), + package: String::new(), + binary: String::new(), + quality: Quality::Fast, + placement: Placement::Auto, + target_root: None, + offline: false, + features: None, + no_default_features: false, + reference_artifact: None, + max_size_ratio: 1.5, + size_basis: SizeBasis::Unmodified, + object_backend: ObjectBackend::Native, + bitcode_wrapper: None, + tpde_controller: None, + tpde_bridge: None, + llvm_llc: None, + allow_backend_fallback: false, + gpu_emitter: None, + gpu_library: None, + gpu_shader: None, + gpu_backend: None, + gpu_device: 0, + } + } +} + +#[derive(Debug, Serialize)] +pub struct BackendTool { + pub path: PathBuf, + pub sha256: String, +} + +#[derive(Debug, Serialize)] +pub struct BackendToolchain { + pub provenance_sha256: String, + pub wrapper: BackendTool, + pub controller: BackendTool, + pub bridge: Option, + pub llc: Option, + pub allow_fallback: bool, + pub gpu_emitter: Option, + pub gpu_library: Option, + pub gpu_shader: Option, + pub gpu_backend: Option, + pub gpu_device: u32, + pub rust_llvm_library: BackendTool, + pub llc_version: Option, + pub fallback_library_selection_verified: bool, + pub fallback_library_scope: &'static str, +} + +#[derive(Debug, Default, Serialize)] +pub struct BackendEvidence { + pub directory: PathBuf, + pub receipt_count: u64, + pub successful_receipts: u64, + pub failed_receipts: u64, + pub failed_passthrough_probes: u64, + pub failed_semantic_invocations: u64, + pub forwarded_passthrough: u64, + pub translated_units: u64, + pub unit_backend_counts: BTreeMap, + pub fallback_units: u64, + pub fallback_stage_counts: BTreeMap, + pub fallback_reasons: BTreeMap, + pub physical_gpu_units: u64, + pub gpu_kernel_dispatches: u64, + pub gpu_devices: BTreeMap, + pub frontend_ms: f64, + pub materialize_ms: f64, + pub emit_ms: f64, + pub link_ms: f64, + pub wrapper_total_ms: f64, + pub timing_scope: &'static str, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CachedGpuLineage { + schema: String, + version: u32, + pub artifact_path: PathBuf, + pub artifact_sha256: String, + pub backend_provenance_sha256: String, + pub rustc_version_verbose: String, + pub quality: Quality, + pub manifest_sha256: String, + pub lockfile_sha256: String, + pub rustflags_sha256: Option, + pub encoded_rustflags_sha256: Option, + pub qualified_receipts_directory: PathBuf, + pub qualified_receipt_hashes: BTreeMap, + pub physical_gpu_units: u64, + pub gpu_devices: BTreeMap, + pub origin_backend: String, +} + +#[derive(Debug, Serialize)] +pub struct Source { + pub manifest_path: PathBuf, + pub workspace_root: PathBuf, + pub manifest_sha256: String, + pub lockfile_sha256: String, + pub git_head: Option, + pub git_worktree_dirty: Option, +} + +#[derive(Debug, Serialize)] +pub struct Artifact { + pub path: PathBuf, + pub cargo_fresh: bool, + pub raw_bytes: u64, + pub measured_bytes: u64, + pub size_basis: SizeBasis, + pub measurement_command: Vec, + pub sha256: String, + pub reference: Option, +} + +#[derive(Debug, Serialize)] +pub struct Reference { + pub path: PathBuf, + pub raw_bytes: u64, + pub measured_bytes: u64, + pub sha256: String, + pub max_ratio: f64, + pub observed_ratio: f64, + pub within_limit: bool, +} + +#[derive(Debug, Serialize)] +pub struct Report { + pub schema: &'static str, + pub version: u32, + pub status: &'static str, + pub quality: Quality, + pub requested_placement: Placement, + pub object_backend: ObjectBackend, + pub backend_toolchain: Option, + pub backend_environment: BTreeMap, + pub object_backend_receipts: Option, + pub planned_backend: &'static str, + pub actual_backend: Option<&'static str>, + pub gpu_compilation_jobs: u64, + pub gpu_executed: bool, + pub current_run_gpu_qualified: bool, + pub cache_only: bool, + pub cached_gpu_lineage: Option, + pub performance_speedup_demonstrated: bool, + pub placement_reason: &'static str, + pub toolchain: &'static str, + pub rustc_version_verbose: String, + pub rustc_path: PathBuf, + pub source: Source, + pub cargo_program: &'static str, + pub cargo_arguments: Vec, + pub target_dir: PathBuf, + pub invocation_directory: PathBuf, + pub jobserver_inherited: bool, + pub rustflags_sha256: Option, + pub encoded_rustflags_sha256: Option, + pub semantic_policy: &'static str, + pub policy_scope: &'static str, + pub cargo_elapsed_ms: Option, + pub total_elapsed_ms: f64, + pub artifact: Option, + pub error: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Owner { + version: u32, + manifest_path: PathBuf, + quality: Quality, + toolchain: String, + object_backend: ObjectBackend, + backend_provenance: String, +} + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn file_digest(path: &Path) -> Result { + fs::read(path) + .map(|bytes| digest(&bytes)) + .map_err(|error| format!("{}: {error}", path.display())) +} + +fn backend_tool(path: &Option, name: &str) -> Result { + let path = path + .as_ref() + .ok_or_else(|| format!("Alternate object backend requires {name}"))?; + let path = fs::canonicalize(path).map_err(|error| format!("{name}: {error}"))?; + if !path.is_file() { + return Err(format!("{name} must identify a regular tool file")); + } + Ok(BackendTool { + sha256: file_digest(&path)?, + path, + }) +} + +fn pinned_llvm_library(sysroot: &Path) -> Result { + let directory = sysroot.join("lib"); + let mut paths = Vec::new(); + let mut matched = Vec::new(); + let mut ignored = Vec::new(); + for entry in fs::read_dir(&directory).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name == "libLLVM.dylib" + || name == "libLLVM.so" + || (name.starts_with("libLLVM") && (name.ends_with(".so") || name.contains(".so."))) + { + let path = fs::canonicalize(entry.path()).map_err(|e| e.to_string())?; + matched.push(name.into_owned()); + // Official Linux Rust also ships a .so-named GNU linker script. + // dlopen needs its ELF target, so filenames alone cannot select a + // compiler runtime library. Keep distinct native images ambiguous. + let mut magic = [0u8; 4]; + let bytes = fs::File::open(&path) + .and_then(|mut file| file.read(&mut magic)) + .map_err(|e| format!("Inspect shared-library candidate {}: {e}", path.display()))?; + if bytes != 4 + || !matches!( + magic, + [0x7f, b'E', b'L', b'F'] + | [0xce, 0xfa, 0xed, 0xfe] + | [0xcf, 0xfa, 0xed, 0xfe] + | [0xfe, 0xed, 0xfa, 0xce] + | [0xfe, 0xed, 0xfa, 0xcf] + | [0xca, 0xfe, 0xba, 0xbe] + | [0xca, 0xfe, 0xba, 0xbf] + | [0xbe, 0xba, 0xfe, 0xca] + | [0xbf, 0xba, 0xfe, 0xca] + ) + { + ignored.push(path); + continue; + } + if !paths.contains(&path) { + paths.push(path); + } + } + } + if paths.len() != 1 { + matched.sort(); + paths.sort(); + ignored.sort(); + return Err(format!("Pinned sysroot must identify exactly one native Rust LLVM shared library in {}; matched filenames {:?}; native candidates {} {:?}; ignored non-native files {:?}",directory.display(),matched,paths.len(),paths,ignored)); + } + backend_tool(&Some(paths.remove(0)), "pinned Rust LLVM shared library") +} + +fn backend_toolchain( + options: &Options, + sysroot: &Path, +) -> Result, String> { + if options.object_backend == ObjectBackend::Native { + if options.bitcode_wrapper.is_some() + || options.tpde_controller.is_some() + || options.tpde_bridge.is_some() + || options.llvm_llc.is_some() + || options.allow_backend_fallback + || options.gpu_emitter.is_some() + || options.gpu_library.is_some() + || options.gpu_shader.is_some() + || options.gpu_backend.is_some() + || options.gpu_device != 0 + { + return Err( + "Backend tool/fallback options require an alternate --object-backend".into(), + ); + } + return Ok(None); + } + if options.quality == Quality::Release { + return Err("Alternate object backends support fast or balanced only; release preserves the native Cargo LTO contract".into()); + } + let wrapper = backend_tool(&options.bitcode_wrapper, "--bitcode-wrapper")?; + let controller = backend_tool(&options.tpde_controller, "--tpde-controller")?; + let bridge = backend_tool(&options.tpde_bridge, "--tpde-bridge")?; + let llc = backend_tool(&options.llvm_llc, "--llvm-llc")?; + let rust_llvm_library = pinned_llvm_library(sysroot)?; + let llc_version = Command::new(&llc.path) + .arg("--version") + .env("RUST_LLVM_LIBRARY", &rust_llvm_library.path) + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| String::from_utf8(output.stdout).ok()); + let fallback_library_selection_verified = llc_version.as_ref().is_some_and(|version| { + version.starts_with("Rust LLVM C-API object emitter\n") + && version + .lines() + .find_map(|line| line.strip_prefix("Existing Rust LLVM library: ")) + .and_then(|path| fs::canonicalize(path).ok()) + .is_some_and(|path| path == rust_llvm_library.path) + && version.lines().any(|line| line == "LLVM version 22.1.8") + }); + let fallback_library_scope = if fallback_library_selection_verified { + "Pinned Rust LLVM C-API library selected and hashed" + } else { + "External llc executable hashed; additional dynamic dependencies are unqualified, so GPU cache lineage is unavailable" + }; + let (gpu_emitter, gpu_library, gpu_shader, gpu_backend) = + if options.object_backend == ObjectBackend::Gpu { + let backend = options.gpu_backend.unwrap_or({ + if matches!(options.placement, Placement::Metal | Placement::Cuda) { + options.placement + } else if cfg!(target_os = "macos") { + Placement::Metal + } else { + Placement::Cuda + } + }); + if !matches!(backend, Placement::Metal | Placement::Cuda) + || options.placement == Placement::Cpu + || (matches!(options.placement, Placement::Metal | Placement::Cuda) + && options.placement != backend) + { + return Err( + "GPU object backend requires matching Metal or CUDA placement/backend".into(), + ); + } + ( + Some(backend_tool(&options.gpu_emitter, "--gpu-emitter")?), + Some(backend_tool(&options.gpu_library, "--gpu-library")?), + if backend == Placement::Metal { + Some(backend_tool(&options.gpu_shader, "--gpu-shader")?) + } else { + options + .gpu_shader + .as_ref() + .map(|_| backend_tool(&options.gpu_shader, "--gpu-shader")) + .transpose()? + }, + Some(backend), + ) + } else { + if options.gpu_emitter.is_some() + || options.gpu_library.is_some() + || options.gpu_shader.is_some() + || options.gpu_backend.is_some() + || options.gpu_device != 0 + { + return Err("GPU tool options require --object-backend gpu".into()); + } + (None, None, None, None) + }; + let provenance = serde_json::to_vec(&( + options.object_backend, + &wrapper, + &controller, + &bridge, + &llc, + options.allow_backend_fallback, + &gpu_emitter, + &gpu_library, + &gpu_shader, + gpu_backend, + options.gpu_device, + &rust_llvm_library, + &llc_version, + fallback_library_selection_verified, + )) + .map_err(|error| error.to_string())?; + Ok(Some(BackendToolchain { + provenance_sha256: digest(&provenance), + wrapper, + controller, + bridge: Some(bridge), + llc: Some(llc), + allow_fallback: options.allow_backend_fallback, + gpu_emitter, + gpu_library, + gpu_shader, + gpu_backend, + gpu_device: options.gpu_device, + rust_llvm_library, + llc_version, + fallback_library_selection_verified, + fallback_library_scope, + })) +} + +fn aggregate_receipts(directory: &Path, options: &Options) -> Result { + let mut evidence = BackendEvidence {directory:directory.to_owned(), timing_scope:"Sum of wrapper invocation stages; Cargo processes can overlap, so these are not whole-build wall-time shares", ..Default::default()}; + let mut files = fs::read_dir(directory) + .map_err(|e| e.to_string())? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(|e| e.to_string())?; + files.sort(); + for file in files { + if file.extension().is_none_or(|extension| extension != "json") { + continue; + } + let receipt: serde_json::Value = + serde_json::from_slice(&fs::read(&file).map_err(|e| e.to_string())?) + .map_err(|error| format!("Invalid backend receipt {}: {error}", file.display()))?; + if receipt["schema"] != "gpu-rust-object-bridge" + || receipt["version"] != 1 + || receipt["requested_backend"] != options.object_backend.controller_name() + || receipt["quality"] != options.quality.name() + { + return Err(format!( + "Backend receipt schema, quality, placement, or request provenance mismatch: {}", + file.display() + )); + } + match receipt["status"].as_str() { + Some("success" | "passthrough") => evidence.successful_receipts += 1, + Some("failed") => evidence.failed_receipts += 1, + _ => return Err("Backend receipt has an unknown status".into()), + } + evidence.receipt_count += 1; + let executed = receipt["gpu_executed"] + .as_bool() + .ok_or("Backend receipt omitted GPU execution state")?; + if receipt["performance_speedup_demonstrated"] != false { + return Err( + "Object receipt cannot infer a performance speedup from device execution".into(), + ); + } + let mut receipt_gpu_units = 0; + let forwarded = receipt["forwarded_passthrough"] + .as_bool() + .ok_or("Backend receipt omitted passthrough state")?; + let units = receipt["units"] + .as_array() + .ok_or("Backend receipt omitted units")?; + if forwarded { + evidence.forwarded_passthrough += 1; + } + if receipt["status"] == "failed" { + if forwarded && units.is_empty() { + evidence.failed_passthrough_probes += 1; + } else { + evidence.failed_semantic_invocations += 1; + } + } + for (key, total) in [ + ("frontend_ms", &mut evidence.frontend_ms), + ("materialize_ms", &mut evidence.materialize_ms), + ("emit_ms", &mut evidence.emit_ms), + ("link_ms", &mut evidence.link_ms), + ("total_ms", &mut evidence.wrapper_total_ms), + ] { + let value = receipt[key] + .as_f64() + .ok_or_else(|| format!("Backend receipt omitted stage {key}"))?; + if !value.is_finite() || value < 0.0 { + return Err(format!("Backend receipt has invalid stage {key}")); + } + *total += value; + } + for unit in units { + let backend = unit["backend"] + .as_str() + .ok_or("Backend unit omitted actual backend")?; + if !matches!(backend, "tpde" | "llvm" | "metal" | "cuda") { + return Err("Backend unit reports an unknown object emitter".into()); + } + *evidence + .unit_backend_counts + .entry(backend.into()) + .or_default() += 1; + evidence.translated_units += 1; + if matches!(backend, "metal" | "cuda") { + let selected = options.gpu_backend.unwrap_or({ + if matches!(options.placement, Placement::Metal | Placement::Cuda) { + options.placement + } else if cfg!(target_os = "macos") { + Placement::Metal + } else { + Placement::Cuda + } + }); + let expected = if selected == Placement::Metal { + "metal" + } else { + "cuda" + }; + let dispatches = unit["kernel_dispatches"] + .as_u64() + .ok_or("GPU unit omitted dispatch evidence")?; + let device = unit["device_id"] + .as_str() + .filter(|id| !id.is_empty()) + .ok_or("GPU unit omitted physical device identity")?; + if options.object_backend != ObjectBackend::Gpu + || backend != expected + || unit["physical_gpu"] != true + || unit["kernel_output_used"] != true + || dispatches == 0 + { + return Err( + "GPU object unit lacks matching physical execution and used kernel output" + .into(), + ); + } + evidence.physical_gpu_units += 1; + receipt_gpu_units += 1; + evidence.gpu_kernel_dispatches = evidence + .gpu_kernel_dispatches + .checked_add(dispatches) + .ok_or("GPU dispatch total overflow")?; + *evidence.gpu_devices.entry(device.into()).or_default() += 1; + } else if unit["physical_gpu"] == true + || unit["kernel_output_used"] == true + || unit["kernel_dispatches"] + .as_u64() + .is_some_and(|value| value > 0) + { + return Err("CPU object unit reports contradictory GPU execution evidence".into()); + } + if let Some(reason) = unit["fallback_reason"] + .as_str() + .filter(|reason| !reason.is_empty()) + { + if !matches!( + options.object_backend, + ObjectBackend::Tpde | ObjectBackend::Gpu + ) || !matches!(backend, "llvm" | "tpde") + || !options.allow_backend_fallback + { + return Err("Backend receipt records an unauthorized fallback".into()); + } + evidence.fallback_units += 1; + *evidence + .fallback_stage_counts + .entry("object-emission".into()) + .or_default() += 1; + *evidence.fallback_reasons.entry(reason.into()).or_default() += 1; + } else if (options.object_backend == ObjectBackend::Tpde && backend == "llvm") + || (options.object_backend == ObjectBackend::Gpu + && matches!(backend, "llvm" | "tpde")) + { + return Err("CPU fallback unit omitted its explicit reason".into()); + } + } + if executed != (receipt_gpu_units > 0) { + return Err( + "Backend receipt GPU claim disagrees with verified object-unit evidence".into(), + ); + } + } + Ok(evidence) +} + +fn observed_backend(evidence: &BackendEvidence, fresh: bool) -> Option<&'static str> { + if fresh && evidence.translated_units == 0 { + return Some("cached-artifact"); + } + let tpde = evidence + .unit_backend_counts + .get("tpde") + .copied() + .unwrap_or(0) + > 0; + let llvm = evidence + .unit_backend_counts + .get("llvm") + .copied() + .unwrap_or(0) + > 0; + if evidence.physical_gpu_units > 0 { + return Some(if tpde || llvm || evidence.forwarded_passthrough > 0 { + "rustc-bitcode-gpu-and-cpu" + } else if evidence.unit_backend_counts.contains_key("metal") { + "rustc-bitcode-metal" + } else { + "rustc-bitcode-cuda" + }); + } + match (tpde, llvm, evidence.forwarded_passthrough > 0) { + (true, false, false) => Some("rustc-bitcode-tpde-cpu"), + (true, _, _) => Some("rustc-bitcode-mixed-cpu"), + (false, true, _) => Some("rustc-bitcode-llvm-cpu"), + (false, false, true) => Some("rustc-llvm-cpu-passthrough"), + _ if fresh => Some("cached-artifact"), + _ => None, + } +} + +fn receipt_hashes(directory: &Path) -> Result, String> { + let mut hashes = BTreeMap::new(); + for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if entry + .path() + .extension() + .is_none_or(|extension| extension != "json") + { + continue; + } + let name = entry + .file_name() + .into_string() + .map_err(|_| "Receipt filename is not UTF-8")?; + hashes.insert(name, file_digest(&entry.path())?); + } + Ok(hashes) +} + +fn lineage_path(target: &Path) -> PathBuf { + target.join(".gpu-cargo-qualified-gpu-artifact.json") +} + +fn qualified_lineage(report: &Report) -> Result { + let artifact = report + .artifact + .as_ref() + .ok_or("GPU lineage has no artifact")?; + let tools = report + .backend_toolchain + .as_ref() + .ok_or("GPU lineage has no tool provenance")?; + let evidence = report + .object_backend_receipts + .as_ref() + .ok_or("GPU lineage has no receipts")?; + if !report.current_run_gpu_qualified || evidence.physical_gpu_units == 0 { + return Err("Current run has no qualified GPU lineage".into()); + } + if evidence.failed_semantic_invocations > 0 { + return Err( + "Failed semantic backend invocations cannot qualify GPU artifact lineage".into(), + ); + } + if !tools.fallback_library_selection_verified { + return Err("GPU cache lineage requires a qualified selected fallback LLVM library".into()); + } + Ok(CachedGpuLineage { + schema: "gpu-cargo-qualified-gpu-artifact".into(), + version: 1, + artifact_path: artifact.path.clone(), + artifact_sha256: artifact.sha256.clone(), + backend_provenance_sha256: tools.provenance_sha256.clone(), + rustc_version_verbose: report.rustc_version_verbose.clone(), + quality: report.quality, + manifest_sha256: report.source.manifest_sha256.clone(), + lockfile_sha256: report.source.lockfile_sha256.clone(), + rustflags_sha256: report.rustflags_sha256.clone(), + encoded_rustflags_sha256: report.encoded_rustflags_sha256.clone(), + qualified_receipts_directory: evidence.directory.clone(), + qualified_receipt_hashes: receipt_hashes(&evidence.directory)?, + physical_gpu_units: evidence.physical_gpu_units, + gpu_devices: evidence.gpu_devices.clone(), + origin_backend: report + .actual_backend + .ok_or("GPU lineage omitted actual backend")? + .into(), + }) +} + +fn save_lineage(report: &Report) -> Result<(), String> { + let lineage = qualified_lineage(report)?; + let path = lineage_path(&report.target_dir); + let temporary = path.with_extension(format!("{}.tmp", std::process::id())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary) + .map_err(|e| e.to_string())?; + let result = file + .write_all(&serde_json::to_vec(&lineage).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string()); + drop(file); + let result = result.and_then(|_| fs::rename(&temporary, path).map_err(|e| e.to_string())); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result +} + +fn cached_lineage(report: &Report, options: &Options) -> Result { + let lineage: CachedGpuLineage = serde_json::from_slice( + &fs::read(lineage_path(&report.target_dir)) + .map_err(|e| format!("No qualified cached GPU lineage: {e}"))?, + ) + .map_err(|e| format!("Invalid cached GPU lineage: {e}"))?; + let artifact = report + .artifact + .as_ref() + .ok_or("Cached GPU lineage has no artifact")?; + let tools = report + .backend_toolchain + .as_ref() + .ok_or("Cached GPU lineage has no tools")?; + if lineage.schema != "gpu-cargo-qualified-gpu-artifact" + || lineage.version != 1 + || lineage.physical_gpu_units == 0 + || lineage.artifact_path != artifact.path + || lineage.artifact_sha256 != artifact.sha256 + || lineage.backend_provenance_sha256 != tools.provenance_sha256 + || lineage.rustc_version_verbose != report.rustc_version_verbose + || lineage.quality != report.quality + || lineage.manifest_sha256 != report.source.manifest_sha256 + || lineage.lockfile_sha256 != report.source.lockfile_sha256 + || lineage.rustflags_sha256 != report.rustflags_sha256 + || lineage.encoded_rustflags_sha256 != report.encoded_rustflags_sha256 + || !tools.fallback_library_selection_verified + || !lineage + .qualified_receipts_directory + .starts_with(report.target_dir.join(".gpu-link-receipts")) + { + return Err("Cached GPU artifact/input/tool provenance does not match".into()); + } + if lineage.qualified_receipt_hashes.is_empty() + || receipt_hashes(&lineage.qualified_receipts_directory)? + != lineage.qualified_receipt_hashes + { + return Err("Cached GPU receipt lineage changed or is incomplete".into()); + } + let evidence = aggregate_receipts(&lineage.qualified_receipts_directory, options)?; + if evidence.physical_gpu_units != lineage.physical_gpu_units + || evidence.gpu_devices != lineage.gpu_devices + || evidence.failed_semantic_invocations > 0 + { + return Err("Cached GPU execution evidence does not qualify its lineage".into()); + } + Ok(lineage) +} + +fn text_path(path: &Path) -> Result { + path.to_str() + .map(str::to_owned) + .ok_or_else(|| format!("Path must be UTF-8: {}", path.display())) +} + +fn rustup_command(program: &str) -> Command { + let mut command = Command::new("rustup"); + command.args(["run", TOOLCHAIN, program]); + command +} + +fn output_text(mut command: Command, label: &str) -> Result { + let result = command + .output() + .map_err(|error| format!("Could not run {label}: {error}"))?; + if !result.status.success() { + return Err(format!( + "{label} failed: {}", + String::from_utf8_lossy(&result.stderr) + )); + } + String::from_utf8(result.stdout).map_err(|_| format!("{label} emitted non-UTF-8 output")) +} + +fn git_output(root: &Path, arguments: &[&str]) -> Option { + let output = Command::new("git") + .arg("-C") + .arg(root) + .args(arguments) + .output() + .ok()?; + output.status.success().then(|| { + String::from_utf8_lossy(&output.stdout) + .trim_end() + .to_owned() + }) +} + +fn absolute(path: &Path) -> Result { + if path.is_absolute() { + Ok(path.to_owned()) + } else { + env::current_dir() + .map(|current| current.join(path)) + .map_err(|error| error.to_string()) + } +} + +fn claim_target(path: &Path, owner: &Owner) -> Result<(), String> { + let marker = path.join(".gpu-cargo-owner.json"); + if marker.exists() { + let existing: Owner = + serde_json::from_slice(&fs::read(&marker).map_err(|e| e.to_string())?) + .map_err(|e| format!("Invalid target owner marker: {e}"))?; + if existing != *owner { + return Err(format!( + "Target directory belongs to a different policy: {}", + path.display() + )); + } + return Ok(()); + } + if path.exists() + && fs::read_dir(path) + .map_err(|error| error.to_string())? + .next() + .is_some() + { + return Err(format!( + "Refusing to adopt a nonempty target directory without an owner marker: {}", + path.display() + )); + } + fs::create_dir_all(path).map_err(|error| error.to_string())?; + let mut marker_file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&marker) + .map_err(|error| format!("Could not create target owner marker: {error}"))?; + marker_file + .write_all(&serde_json::to_vec(owner).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string()) +} + +fn strip_arguments() -> Result, String> { + if cfg!(target_os = "macos") { + Ok(vec!["-S".into(), "-x".into()]) + } else if cfg!(target_os = "linux") { + Ok(vec!["--strip-debug".into(), "--discard-all".into()]) + } else { + Err("strip-debug measurement supports macOS and Linux only".into()) + } +} + +fn measure(path: &Path, basis: SizeBasis, directory: &Path, name: &str) -> Result { + if matches!(basis, SizeBasis::Unmodified) { + return fs::metadata(path) + .map(|meta| meta.len()) + .map_err(|e| e.to_string()); + } + fs::create_dir_all(directory).map_err(|e| e.to_string())?; + let copy = directory.join(format!("{}-{name}", std::process::id())); + fs::copy(path, ©).map_err(|e| e.to_string())?; + let result = Command::new("strip") + .args(strip_arguments()?) + .arg(©) + .output(); + let measured = match result { + Ok(result) if result.status.success() => fs::metadata(©) + .map(|m| m.len()) + .map_err(|e| e.to_string()), + Ok(result) => Err(format!( + "strip measurement failed: {}", + String::from_utf8_lossy(&result.stderr) + )), + Err(error) => Err(format!("Could not run strip: {error}")), + }; + let _ = fs::remove_file(copy); + measured +} + +/// Build with the stock pinned compiler or produce an equivalent command plan. +/// Environment and inherited descriptors are retained for Cargo's jobserver. +pub fn execute(options: &Options) -> Result { + let start = Instant::now(); + if matches!(options.placement, Placement::Metal | Placement::Cuda) + && options.object_backend != ObjectBackend::Gpu + { + return Err("Required GPU placement is unavailable for this object backend; select a qualified GPU object adapter explicitly".into()); + } + if !options.max_size_ratio.is_finite() || options.max_size_ratio < 1.0 { + return Err("Maximum size ratio must be finite and at least 1.0".into()); + } + for value in [&options.package, &options.binary] { + if value.is_empty() || value.starts_with('-') { + return Err( + "Package and binary names must be nonempty and cannot start with '-'".into(), + ); + } + } + for key in ["RUSTC", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER"] { + if env::var_os(key).is_some_and(|value| !value.is_empty()) { + return Err(format!( + "{key} overrides the pinned compiler; unset it for this experiment" + )); + } + } + for key in ["RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS"] { + if env::var(key).is_ok_and(|value| value.contains("codegen-backend")) { + return Err(format!( + "{key} requests a different backend; the report requires stock LLVM" + )); + } + if options.object_backend != ObjectBackend::Native + && env::var(key).is_ok_and(|value| { + value.contains("lto=") + || value.contains("linker-plugin-lto") + || value.contains("embed-bitcode") + }) + { + return Err(format!( + "{key} overrides the bitcode adapter's required LTO/embedding policy" + )); + } + } + if options.object_backend != ObjectBackend::Native && env::var_os("RUST_LLVM_LIBRARY").is_some() + { + return Err( + "RUST_LLVM_LIBRARY overrides the pinned fallback library; unset it for this experiment" + .into(), + ); + } + let manifest = + fs::canonicalize(&options.manifest_path).map_err(|error| format!("Manifest: {error}"))?; + let mut metadata = rustup_command("cargo"); + metadata + .args([ + "metadata", + "--locked", + "--no-deps", + "--format-version", + "1", + "--manifest-path", + ]) + .arg(&manifest); + if options.offline { + metadata.arg("--offline"); + } + let metadata: serde_json::Value = + serde_json::from_str(&output_text(metadata, "Cargo metadata")?) + .map_err(|error| format!("Invalid Cargo metadata: {error}"))?; + let workspace = PathBuf::from( + metadata["workspace_root"] + .as_str() + .ok_or("Cargo omitted workspace root")?, + ); + let mut version_command = rustup_command("rustc"); + version_command.arg("-vV"); + let version = output_text(version_command, "rustc version")?; + if !version.starts_with(&format!("rustc {TOOLCHAIN} ")) { + return Err(format!("Expected rustc {TOOLCHAIN}")); + } + let mut sysroot_command = rustup_command("rustc"); + sysroot_command.args(["--print", "sysroot"]); + let sysroot = output_text(sysroot_command, "rustc sysroot")?; + let backend_tools = backend_toolchain(options, Path::new(sysroot.trim()))?; + let rustc_path = + fs::canonicalize(Path::new(sysroot.trim()).join("bin/rustc")).map_err(|e| e.to_string())?; + let target_root = absolute( + &options + .target_root + .clone() + .unwrap_or_else(|| workspace.join("target/gpu-cargo")), + )?; + let backend_component = backend_tools + .as_ref() + .map(|tools| { + format!( + "{}-{}", + options.object_backend.name(), + &tools.provenance_sha256[..16] + ) + }) + .unwrap_or_else(|| "native".into()); + let target_dir = target_root + .join(options.quality.name()) + .join(backend_component); + let mut backend_environment = BTreeMap::new(); + let receipts_directory = if let Some(tools) = &backend_tools { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| e.to_string())? + .as_nanos(); + let directory = target_dir + .join(".gpu-link-receipts") + .join(format!("run-{}-{nonce}", std::process::id())); + backend_environment.insert( + "RUST_LLVM_LIBRARY".into(), + text_path(&tools.rust_llvm_library.path)?, + ); + backend_environment.insert( + "GPU_LINK_CONTROLLER".into(), + text_path(&tools.controller.path)?, + ); + backend_environment.insert( + "GPU_LINK_BRIDGE".into(), + text_path(&tools.bridge.as_ref().ok_or("Bridge omitted")?.path)?, + ); + backend_environment.insert( + "GPU_LINK_LLC".into(), + text_path(&tools.llc.as_ref().ok_or("LLVM llc omitted")?.path)?, + ); + backend_environment.insert( + "GPU_LINK_BACKEND".into(), + options.object_backend.controller_name().into(), + ); + backend_environment.insert( + "GPU_LINK_ALLOW_FALLBACK".into(), + if tools.allow_fallback { "1" } else { "0" }.into(), + ); + backend_environment.insert("GPU_LINK_QUALITY".into(), options.quality.name().into()); + backend_environment.insert("GPU_LINK_RECEIPTS".into(), text_path(&directory)?); + if let Some(emitter) = &tools.gpu_emitter { + backend_environment.insert("GPU_LINK_GPU_EMITTER".into(), text_path(&emitter.path)?); + backend_environment.insert( + "GPU_LINK_GPU_LIBRARY".into(), + text_path( + &tools + .gpu_library + .as_ref() + .ok_or("GPU library omitted")? + .path, + )?, + ); + backend_environment.insert( + "GPU_LINK_GPU_BACKEND".into(), + match tools.gpu_backend { + Some(Placement::Metal) => "metal", + Some(Placement::Cuda) => "cuda", + _ => return Err("GPU backend omitted".into()), + } + .into(), + ); + backend_environment.insert( + "GPU_LINK_GPU_SHADER".into(), + tools + .gpu_shader + .as_ref() + .map(|shader| text_path(&shader.path)) + .transpose()? + .unwrap_or_default(), + ); + backend_environment.insert("GPU_LINK_GPU_DEVICE".into(), tools.gpu_device.to_string()); + } + Some(directory) + } else { + None + }; + let source = Source { + manifest_path: manifest.clone(), + workspace_root: workspace.clone(), + manifest_sha256: file_digest(&manifest)?, + lockfile_sha256: file_digest(&workspace.join("Cargo.lock"))?, + git_head: git_output(&workspace, &["rev-parse", "HEAD"]), + git_worktree_dirty: git_output( + &workspace, + &["status", "--porcelain", "--untracked-files=normal"], + ) + .map(|s| !s.is_empty()), + }; + let mut arguments = vec![ + "run".into(), + TOOLCHAIN.into(), + "cargo".into(), + "build".into(), + "--locked".into(), + "--release".into(), + "--manifest-path".into(), + text_path(&manifest)?, + "--package".into(), + options.package.clone(), + "--bin".into(), + options.binary.clone(), + "--target-dir".into(), + text_path(&target_dir)?, + "--message-format=json-render-diagnostics".into(), + ]; + // A project can configure another compiler. Pin the executable used by + // Cargo rather than only probing rustup's selected toolchain. + arguments.extend([ + "--config".into(), + format!( + "build.rustc={}", + serde_json::to_string(&text_path(&rustc_path)?).map_err(|e| e.to_string())? + ), + "--config".into(), + format!( + "build.rustc-wrapper={}", + serde_json::to_string( + &backend_tools + .as_ref() + .map(|tools| text_path(&tools.wrapper.path)) + .transpose()? + .unwrap_or_default() + ) + .map_err(|e| e.to_string())? + ), + "--config".into(), + "build.rustc-workspace-wrapper=\"\"".into(), + ]); + if options.offline { + arguments.push("--offline".into()); + } + if options.no_default_features { + arguments.push("--no-default-features".into()); + } + if let Some(features) = &options.features { + arguments.extend(["--features".into(), features.clone()]); + } + for config in options.quality.config() { + arguments.extend(["--config".into(), config]); + } + let mut report = Report { + schema: "gpu-cargo-report", version: 1, status: "planned", quality: options.quality, + requested_placement: options.placement, planned_backend: options.object_backend.planned(), actual_backend: None, + object_backend:options.object_backend,backend_toolchain:backend_tools,backend_environment,object_backend_receipts:None, + gpu_compilation_jobs: 0, + gpu_executed:false,current_run_gpu_qualified:false,cache_only:false,cached_gpu_lineage:None, + performance_speedup_demonstrated:false, + placement_reason: "Actual object emitters and GPU work are verified from wrapper receipts; native policies use pinned stock CPU LLVM", + toolchain: TOOLCHAIN, rustc_version_verbose: version, rustc_path, source, + cargo_program: "rustup", cargo_arguments: arguments.clone(), target_dir: target_dir.clone(), + invocation_directory: env::current_dir().map_err(|e| e.to_string())?, + jobserver_inherited: env::var_os("CARGO_MAKEFLAGS").is_some() || env::var_os("MAKEFLAGS").is_some(), + rustflags_sha256: env::var("RUSTFLAGS").ok().map(|v| digest(v.as_bytes())), + encoded_rustflags_sha256: env::var("CARGO_ENCODED_RUSTFLAGS").ok().map(|v| digest(v.as_bytes())), + semantic_policy: "Preserve project panic, overflow, debug, features, target, and inherited semantic flags", + policy_scope: "Root release profile; existing package-specific profile overrides and inherited compiler flags retain Cargo precedence", + cargo_elapsed_ms: None, total_elapsed_ms: 0.0, artifact: None, error: None, + }; + let reference = options + .reference_artifact + .as_deref() + .map(fs::canonicalize) + .transpose() + .map_err(|e| format!("Reference artifact: {e}"))?; + if let Some(path) = &reference { + if !path.is_file() || fs::metadata(path).map_err(|e| e.to_string())?.len() == 0 { + return Err("Reference artifact must be a nonempty regular file".into()); + } + let resolved_target = fs::canonicalize(&target_dir).unwrap_or_else(|_| target_dir.clone()); + if path.starts_with(&resolved_target) { + return Err("Reference artifact must be outside the selected quality's target directory so the build cannot replace its baseline".into()); + } + } + if options.plan { + report.total_elapsed_ms = start.elapsed().as_secs_f64() * 1000.0; + return Ok(report); + } + claim_target( + &target_dir, + &Owner { + version: 2, + manifest_path: manifest, + quality: options.quality, + toolchain: TOOLCHAIN.into(), + object_backend: options.object_backend, + backend_provenance: report + .backend_toolchain + .as_ref() + .map(|tools| tools.provenance_sha256.clone()) + .unwrap_or_else(|| "native".into()), + }, + )?; + if let Some(directory) = &receipts_directory { + fs::create_dir_all(directory).map_err(|e| e.to_string())?; + } + let cargo_start = Instant::now(); + let mut cargo = Command::new("rustup"); + cargo.args(&arguments); + for key in [ + "RUST_LLVM_LIBRARY", + "GPU_LINK_CONTROLLER", + "GPU_LINK_BRIDGE", + "GPU_LINK_LLC", + "GPU_LINK_BACKEND", + "GPU_LINK_ALLOW_FALLBACK", + "GPU_LINK_QUALITY", + "GPU_LINK_RECEIPTS", + "GPU_LINK_GPU_EMITTER", + "GPU_LINK_GPU_LIBRARY", + "GPU_LINK_GPU_BACKEND", + "GPU_LINK_GPU_SHADER", + "GPU_LINK_GPU_DEVICE", + ] { + cargo.env_remove(key); + } + cargo.envs(&report.backend_environment); + let mut child = cargo + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|e| format!("Cargo: {e}"))?; + let mut executable = None; + let mut fresh = false; + for line in BufReader::new(child.stdout.take().ok_or("Cargo stdout unavailable")?).lines() { + let line = line.map_err(|e| e.to_string())?; + let message: serde_json::Value = match serde_json::from_str(&line) { + Ok(message) => message, + Err(_) => { + eprintln!("{line}"); + continue; + } + }; + if message["reason"] == "compiler-message" { + if let Some(rendered) = message["message"]["rendered"].as_str() { + eprint!("{rendered}"); + } + } + if message["reason"] == "compiler-artifact" && message["target"]["name"] == options.binary { + if let Some(path) = message["executable"].as_str() { + executable = Some(PathBuf::from(path)); + fresh = message["fresh"].as_bool().unwrap_or(false); + } + } + } + let status = child.wait().map_err(|e| e.to_string())?; + report.cargo_elapsed_ms = Some(cargo_start.elapsed().as_secs_f64() * 1000.0); + report.actual_backend = if let Some(directory) = &receipts_directory { + let evidence = aggregate_receipts(directory, options)?; + let actual = observed_backend(&evidence, fresh); + report.gpu_compilation_jobs = evidence.physical_gpu_units; + report.object_backend_receipts = Some(evidence); + actual + } else { + Some(BACKEND) + }; + report.gpu_executed = report.gpu_compilation_jobs > 0; + report.current_run_gpu_qualified = status.success() + && report.gpu_executed + && report + .object_backend_receipts + .as_ref() + .is_none_or(|evidence| evidence.failed_semantic_invocations == 0); + report.cache_only = fresh + && report + .object_backend_receipts + .as_ref() + .is_none_or(|evidence| evidence.translated_units == 0); + if !status.success() { + report.status = "build-failed"; + report.error = Some(format!("Cargo exited with {status}")); + report.total_elapsed_ms = start.elapsed().as_secs_f64() * 1000.0; + return Ok(report); + } + if report.actual_backend.is_none() { + report.status = "backend-evidence-missing"; + report.error = Some( + "Cargo succeeded without object-backend receipts or a qualified cached artifact".into(), + ); + report.total_elapsed_ms = start.elapsed().as_secs_f64() * 1000.0; + return Ok(report); + } + let path = + fs::canonicalize(executable.ok_or("Cargo did not report the requested binary artifact")?) + .map_err(|e| e.to_string())?; + let measurement_dir = target_dir.join(".gpu-cargo-measurement"); + let measured_bytes = measure(&path, options.size_basis, &measurement_dir, "candidate")?; + let reference = reference + .map(|reference_path| { + let bytes = measure( + &reference_path, + options.size_basis, + &measurement_dir, + "reference", + )?; + let ratio = measured_bytes as f64 / bytes as f64; + Ok::<_, String>(Reference { + raw_bytes: fs::metadata(&reference_path) + .map_err(|e| e.to_string())? + .len(), + measured_bytes: bytes, + sha256: file_digest(&reference_path)?, + path: reference_path, + max_ratio: options.max_size_ratio, + observed_ratio: ratio, + within_limit: ratio <= options.max_size_ratio, + }) + }) + .transpose()?; + let within_limit = reference.as_ref().is_none_or(|r| r.within_limit); + report.artifact = Some(Artifact { + raw_bytes: fs::metadata(&path).map_err(|e| e.to_string())?.len(), + measured_bytes, + size_basis: options.size_basis, + measurement_command: match options.size_basis { + SizeBasis::Unmodified => vec![], + SizeBasis::StripDebug => { + let mut args = vec!["strip".into()]; + args.extend(strip_arguments()?); + args + } + }, + sha256: file_digest(&path)?, + path, + cargo_fresh: fresh, + reference, + }); + report.status = if within_limit { + "ok" + } else { + "size-limit-exceeded" + }; + if !within_limit { + report.error = Some("Generated executable exceeds the declared reference size limit; no fallback or size-changing rebuild was performed".into()); + } + if options.object_backend == ObjectBackend::Gpu { + if report + .object_backend_receipts + .as_ref() + .is_some_and(|evidence| evidence.failed_semantic_invocations > 0) + { + let path = lineage_path(&report.target_dir); + if path.exists() { + fs::remove_file(path).map_err(|e| e.to_string())?; + } + report.status = "backend-receipt-failure"; + report.error=Some("Failed semantic backend invocations prevent GPU qualification; only failed empty passthrough probes may be ignored".into()); + } else if report.current_run_gpu_qualified { + if report + .backend_toolchain + .as_ref() + .is_some_and(|tools| tools.fallback_library_selection_verified) + { + save_lineage(&report)?; + } + } else if report.cache_only { + match cached_lineage(&report, options) { + Ok(lineage) => { + report.cached_gpu_lineage = Some(lineage); + report.actual_backend = Some("cached-artifact"); + } + Err(error) => { + report.status = "unqualified-gpu-cache"; + report.error = Some(error); + } + } + } else { + let path = lineage_path(&report.target_dir); + if path.exists() { + fs::remove_file(path).map_err(|e| e.to_string())?; + } + report.status = "required-gpu-not-executed"; + report.error=Some("This fresh Cargo build emitted no physical GPU modules; CPU fallback does not qualify the required GPU adapter".into()); + } + } + report.total_elapsed_ms = start.elapsed().as_secs_f64() * 1000.0; + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); + + fn receipt(backend: &str, unit: serde_json::Value, executed: bool) -> serde_json::Value { + serde_json::json!({"schema":"gpu-rust-object-bridge","version":1,"status":"success","requested_backend":backend,"quality":"fast","gpu_executed":executed,"performance_speedup_demonstrated":false,"forwarded_passthrough":false,"frontend_ms":2.0,"materialize_ms":1.0,"emit_ms":3.0,"link_ms":4.0,"total_ms":10.0,"units":[unit]}) + } + + fn directory() -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = env::temp_dir().join(format!( + "gpu-cargo-receipt-protocol-{}-{nonce}-{}", + std::process::id(), + NEXT_DIRECTORY.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn pinned_library_discovery_accepts_linux_rust_library_names() { + let path = directory(); + fs::create_dir(path.join("lib")).unwrap(); + let library = path.join("lib/libLLVM-22-rust-1.98.1-stable.so"); + fs::write(&library, b"\x7fELFprotocol library fixture").unwrap(); + let resolved = pinned_llvm_library(&path).unwrap(); + assert_eq!(resolved.path, fs::canonicalize(library).unwrap()); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn pinned_library_discovery_selects_native_llvm_beside_official_linux_linker_script() { + let path = directory(); + fs::create_dir(path.join("lib")).unwrap(); + let library = path.join("lib/libLLVM.so.22.1-rust-1.98.1-stable"); + fs::write(&library, b"\x7fELFprotocol native-library fixture").unwrap(); + fs::write( + path.join("lib/libLLVM-22-rust-1.98.1-stable.so"), + b"INPUT(libLLVM.so.22.1-rust-1.98.1-stable)\n", + ) + .unwrap(); + assert_eq!( + pinned_llvm_library(&path).unwrap().path, + fs::canonicalize(&library).unwrap() + ); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn pinned_library_discovery_rejects_distinct_native_images_with_candidate_diagnostics() { + let path = directory(); + fs::create_dir(path.join("lib")).unwrap(); + fs::write( + path.join("lib/libLLVM.so.22.1-first"), + b"\x7fELFfirst native image fixture", + ) + .unwrap(); + fs::write( + path.join("lib/libLLVM-second.so"), + b"\x7fELFsecond native image fixture", + ) + .unwrap(); + let error = pinned_llvm_library(&path).unwrap_err(); + assert!(error.contains("native candidates 2"), "{error}"); + assert!( + error.contains("libLLVM.so.22.1-first") && error.contains("libLLVM-second.so"), + "{error}" + ); + assert!( + error.contains(path.join("lib").to_str().unwrap()), + "{error}" + ); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn cpu_receipt_protocol_attributes_fallback_and_rejects_a_gpu_claim() { + let path = directory(); + let options = Options { + object_backend: ObjectBackend::Tpde, + allow_backend_fallback: true, + ..Default::default() + }; + let value = receipt( + "tpde", + serde_json::json!({"backend":"llvm","fallback_reason":"Darwin requires LLVM object emission"}), + false, + ); + fs::write( + path.join("invocation.json"), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + let evidence = aggregate_receipts(&path, &options).unwrap(); + assert_eq!(evidence.fallback_units, 1); + assert_eq!(evidence.fallback_stage_counts["object-emission"], 1); + assert_eq!(evidence.frontend_ms, 2.0); + assert_eq!(evidence.wrapper_total_ms, 10.0); + assert_eq!( + observed_backend(&evidence, false), + Some("rustc-bitcode-llvm-cpu") + ); + let mut forged = value; + forged["gpu_executed"] = true.into(); + fs::write( + path.join("invocation.json"), + serde_json::to_vec(&forged).unwrap(), + ) + .unwrap(); + assert!(aggregate_receipts(&path, &options).is_err()); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn gpu_receipt_protocol_rejects_simulation_and_unused_kernel_output() { + let path = directory(); + let options = Options { + object_backend: ObjectBackend::Gpu, + gpu_backend: Some(Placement::Metal), + ..Default::default() + }; + for (physical, used) in [(false, true), (true, false)] { + let value = receipt( + "gpu", + serde_json::json!({"backend":"metal","physical_gpu":physical,"device_id":"protocol simulation","kernel_dispatches":2,"kernel_output_used":used}), + true, + ); + fs::write( + path.join("invocation.json"), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + assert!(aggregate_receipts(&path, &options).is_err()); + } + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn cached_receipt_protocol_preserves_no_new_gpu_execution_and_checks_integrity() { + // Synthetic protocol data checks caching and hash validation. This test + // cannot establish physical GPU execution or a compiler speedup. + let root = directory(); + let previous = root.join(".gpu-link-receipts/previous"); + fs::create_dir_all(&previous).unwrap(); + let options = Options { + object_backend: ObjectBackend::Gpu, + gpu_backend: Some(Placement::Metal), + ..Default::default() + }; + let value = receipt( + "gpu", + serde_json::json!({"backend":"metal","physical_gpu":true,"device_id":"protocol fixture only","kernel_dispatches":2,"kernel_output_used":true}), + true, + ); + let receipt_file = previous.join("invocation.json"); + fs::write(&receipt_file, serde_json::to_vec(&value).unwrap()).unwrap(); + let mut failed_probe = value.clone(); + failed_probe["status"] = "failed".into(); + failed_probe["forwarded_passthrough"] = true.into(); + failed_probe["units"] = serde_json::json!([]); + failed_probe["gpu_executed"] = false.into(); + fs::write( + previous.join("failed-probe.json"), + serde_json::to_vec(&failed_probe).unwrap(), + ) + .unwrap(); + let artifact_path = root.join("fixture-artifact"); + fs::write(&artifact_path, b"protocol fixture artifact").unwrap(); + let mut report = Report { + schema: "gpu-cargo-report", + version: 1, + status: "ok", + quality: Quality::Fast, + requested_placement: Placement::Metal, + object_backend: ObjectBackend::Gpu, + backend_toolchain: Some(BackendToolchain { + provenance_sha256: "fixture provenance".into(), + wrapper: BackendTool { + path: root.join("wrapper"), + sha256: "fixture".into(), + }, + controller: BackendTool { + path: root.join("controller"), + sha256: "fixture".into(), + }, + bridge: None, + llc: None, + allow_fallback: false, + gpu_emitter: None, + gpu_library: None, + gpu_shader: None, + gpu_backend: Some(Placement::Metal), + gpu_device: 0, + rust_llvm_library: BackendTool { + path: root.join("fixture-LLVM"), + sha256: "protocol fixture".into(), + }, + llc_version: None, + fallback_library_selection_verified: true, + fallback_library_scope: "protocol fixture only", + }), + backend_environment: BTreeMap::new(), + object_backend_receipts: Some(aggregate_receipts(&previous, &options).unwrap()), + planned_backend: "fixture", + actual_backend: Some("fixture"), + gpu_compilation_jobs: 1, + gpu_executed: true, + current_run_gpu_qualified: true, + cache_only: false, + cached_gpu_lineage: None, + performance_speedup_demonstrated: false, + placement_reason: "protocol fixture", + toolchain: TOOLCHAIN, + rustc_version_verbose: "fixture rustc".into(), + rustc_path: root.join("rustc"), + source: Source { + manifest_path: root.join("Cargo.toml"), + workspace_root: root.clone(), + manifest_sha256: "fixture manifest".into(), + lockfile_sha256: "fixture lock".into(), + git_head: None, + git_worktree_dirty: None, + }, + cargo_program: "fixture", + cargo_arguments: vec![], + target_dir: root.clone(), + invocation_directory: root.clone(), + jobserver_inherited: false, + rustflags_sha256: None, + encoded_rustflags_sha256: None, + semantic_policy: "fixture", + policy_scope: "fixture", + cargo_elapsed_ms: Some(1.0), + total_elapsed_ms: 1.0, + artifact: Some(Artifact { + path: artifact_path.clone(), + cargo_fresh: true, + raw_bytes: 25, + measured_bytes: 25, + size_basis: SizeBasis::Unmodified, + measurement_command: vec![], + sha256: file_digest(&artifact_path).unwrap(), + reference: None, + }), + error: None, + }; + save_lineage(&report).unwrap(); + assert_eq!( + report + .object_backend_receipts + .as_ref() + .unwrap() + .failed_passthrough_probes, + 1 + ); + assert_eq!( + report + .object_backend_receipts + .as_ref() + .unwrap() + .failed_semantic_invocations, + 0 + ); + report.gpu_compilation_jobs = 0; + report.gpu_executed = false; + report.current_run_gpu_qualified = false; + report.cache_only = true; + report.object_backend_receipts = Some(BackendEvidence::default()); + assert_eq!( + cached_lineage(&report, &options) + .unwrap() + .physical_gpu_units, + 1 + ); + let mut failed_translation = failed_probe.clone(); + failed_translation["forwarded_passthrough"] = false.into(); + fs::write( + previous.join("failed-translation.json"), + serde_json::to_vec(&failed_translation).unwrap(), + ) + .unwrap(); + assert_eq!( + aggregate_receipts(&previous, &options) + .unwrap() + .failed_semantic_invocations, + 1 + ); + // Retain an integrity-matching ledger to prove semantic failure alone + // rejects cached qualification, rather than only a changed file set. + let mut lineage: CachedGpuLineage = + serde_json::from_slice(&fs::read(lineage_path(&root)).unwrap()).unwrap(); + lineage.qualified_receipt_hashes = receipt_hashes(&previous).unwrap(); + fs::write(lineage_path(&root), serde_json::to_vec(&lineage).unwrap()).unwrap(); + assert!(cached_lineage(&report, &options).is_err()); + fs::remove_file(previous.join("failed-translation.json")).unwrap(); + lineage.qualified_receipt_hashes = receipt_hashes(&previous).unwrap(); + fs::write(lineage_path(&root), serde_json::to_vec(&lineage).unwrap()).unwrap(); + assert_eq!(report.gpu_compilation_jobs, 0); + assert!(!report.gpu_executed && !report.current_run_gpu_qualified); + assert!(report.cache_only); + fs::write(&receipt_file, b"changed receipt").unwrap(); + assert!(cached_lineage(&report, &options).is_err()); + fs::write(&receipt_file, serde_json::to_vec(&value).unwrap()).unwrap(); + report.artifact.as_mut().unwrap().sha256 = "changed artifact".into(); + assert!(cached_lineage(&report, &options).is_err()); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/codegen_service.rs b/experiments/gpu-rust-compiler/compiler/src/codegen_service.rs new file mode 100644 index 00000000000..f410dcfdee1 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/codegen_service.rs @@ -0,0 +1,1271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Build-scoped scheduling for immutable scalar emission batches. +//! +//! Schema 1 is an integer, straight-line code-emission fixture, not resolved +//! Rust MIR. Its ABI records describe only scalar i64 arguments and results. +//! The CPU callback runs actual work. Simulated devices can test scheduling but +//! cannot satisfy required hardware placement or publish physical GPU evidence. +//! Client credits are explicitly supplied by the caller; this module does not +//! acquire Cargo jobserver tokens or provide an interprocess daemon. + +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + fmt, + panic::{catch_unwind, AssertUnwindSafe}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Condvar, Mutex, + }, + thread::{self, JoinHandle}, + time::{Duration, Instant}, +}; + +pub const SCHEMA_VERSION: u32 = 1; +static NEXT_SESSION: AtomicU64 = AtomicU64::new(1); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BatchIdentity { + pub workload_id: String, + pub toolchain_id: String, + pub policy_id: String, + pub target_abi: String, + pub source_revision: String, +} + +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct BatchKey { + pub logical_id: String, + pub generation: u64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum CallingConvention { + SystemV = 0, + AppleAarch64 = 1, + WindowsX64 = 2, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Abi { + pub calling_convention: CallingConvention, + pub parameter_count: u32, + pub returns_i64: bool, + pub requires_unwind: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Symbol { + pub name: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Function { + pub symbol: u32, + pub instruction_start: u32, + pub instruction_count: u32, + pub value_count: u32, + pub abi: u32, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u32)] +pub enum Opcode { + Argument = 0, + Const = 1, + Copy = 2, + Add = 3, + Sub = 4, + Mul = 5, + Return = 6, +} + +/// Operand slots not used by the opcode must be zero. Arithmetic is wrapping +/// i64; Argument's immediate is the zero-based scalar argument index. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Instruction { + pub opcode: Opcode, + pub result: Option, + pub operands: [u32; 2], + pub immediate: i64, +} + +/// Private owned arrays enforce validate-once, read-only submission. A caller +/// cannot mutate an accepted input or reuse its storage while work is pending. +#[derive(Debug)] +pub struct FlatBatch { + identity: BatchIdentity, + key: BatchKey, + functions: Box<[Function]>, + instructions: Box<[Instruction]>, + symbols: Box<[Symbol]>, + abis: Box<[Abi]>, + validation: Duration, + owned_bytes: usize, +} + +impl FlatBatch { + pub fn new( + identity: BatchIdentity, + key: BatchKey, + functions: Vec, + instructions: Vec, + symbols: Vec, + abis: Vec, + ) -> Result { + let start = Instant::now(); + for value in [ + &identity.workload_id, + &identity.toolchain_id, + &identity.policy_id, + &identity.target_abi, + &identity.source_revision, + &key.logical_id, + ] { + if value.is_empty() || value.len() > 4096 || value.contains('\0') { + return Err("batch identity must be nonempty, bounded, and contain no NUL".into()); + } + } + if functions.is_empty() + || instructions.is_empty() + || symbols.len() != functions.len() + || abis.is_empty() + { + return Err( + "batch requires functions, instructions, one symbol per function, and ABI records" + .into(), + ); + } + for count in [ + functions.len(), + instructions.len(), + symbols.len(), + abis.len(), + ] { + if u32::try_from(count).is_err() { + return Err("schema index overflow".into()); + } + } + let mut names = HashSet::new(); + for symbol in &symbols { + if symbol.name.is_empty() + || symbol.name.len() > 4096 + || symbol.name.contains('\0') + || !names.insert(symbol.name.as_str()) + { + return Err("symbols must have unique nonempty bounded names".into()); + } + } + let mut function_symbols = HashSet::new(); + let mut next = 0usize; + for function in &functions { + if function.symbol as usize >= symbols.len() + || !function_symbols.insert(function.symbol) + { + return Err("function symbol missing or duplicated".into()); + } + let abi = abis + .get(function.abi as usize) + .ok_or("function ABI index is out of bounds")?; + let begin = function.instruction_start as usize; + let end = begin + .checked_add(function.instruction_count as usize) + .ok_or("instruction range overflow")?; + if begin != next || end > instructions.len() || begin == end { + return Err("function ranges must partition instructions in order".into()); + } + if function.value_count as usize > function.instruction_count as usize { + return Err("value count exceeds available definitions".into()); + } + let mut defined = HashSet::new(); + for (offset, instruction) in instructions[begin..end].iter().enumerate() { + let is_return = instruction.opcode == Opcode::Return; + if is_return != (offset + 1 == end - begin) { + return Err("each function must end with its only Return".into()); + } + let operand_count = match instruction.opcode { + Opcode::Argument | Opcode::Const => 0, + Opcode::Copy => 1, + Opcode::Add | Opcode::Sub | Opcode::Mul => 2, + Opcode::Return => usize::from(abi.returns_i64), + }; + for &value in &instruction.operands[..operand_count] { + if value >= function.value_count || !defined.contains(&value) { + return Err("instruction uses an undefined value".into()); + } + } + if instruction.operands[operand_count..] + .iter() + .any(|&v| v != 0) + { + return Err("unused operand slots must be zero".into()); + } + if is_return { + if instruction.result.is_some() { + return Err("Return cannot define a value".into()); + } + } else { + let value = instruction + .result + .ok_or("instruction must define a value")?; + if value >= function.value_count || !defined.insert(value) { + return Err("result index out of bounds or multiply defined".into()); + } + } + match instruction.opcode { + Opcode::Argument + if instruction.immediate < 0 + || instruction.immediate as u64 >= u64::from(abi.parameter_count) => + { + return Err("Argument index is out of bounds".into()) + } + Opcode::Const | Opcode::Argument => {} + _ if instruction.immediate != 0 => { + return Err("unused immediate must be zero".into()) + } + _ => {} + } + } + if defined.len() != function.value_count as usize { + return Err("value table contains undefined entries".into()); + } + next = end; + } + if next != instructions.len() { + return Err("instructions outside function ranges".into()); + } + let owned_bytes = std::mem::size_of::() + .checked_add(functions.len() * std::mem::size_of::()) + .and_then(|n| n.checked_add(instructions.len() * std::mem::size_of::())) + .and_then(|n| n.checked_add(symbols.len() * std::mem::size_of::())) + .and_then(|n| n.checked_add(abis.len() * std::mem::size_of::())) + .and_then(|n| { + symbols + .iter() + .try_fold(n, |n, s| n.checked_add(s.name.capacity())) + }) + .and_then(|n| { + [ + &identity.workload_id, + &identity.toolchain_id, + &identity.policy_id, + &identity.target_abi, + &identity.source_revision, + &key.logical_id, + ] + .iter() + .try_fold(n, |n, s| n.checked_add(s.capacity())) + }) + .ok_or("owned input size overflow")?; + Ok(Self { + identity, + key, + functions: functions.into_boxed_slice(), + instructions: instructions.into_boxed_slice(), + symbols: symbols.into_boxed_slice(), + abis: abis.into_boxed_slice(), + validation: start.elapsed(), + owned_bytes, + }) + } + pub fn identity(&self) -> &BatchIdentity { + &self.identity + } + pub fn key(&self) -> &BatchKey { + &self.key + } + pub fn functions(&self) -> &[Function] { + &self.functions + } + pub fn instructions(&self) -> &[Instruction] { + &self.instructions + } + pub fn symbols(&self) -> &[Symbol] { + &self.symbols + } + pub fn abis(&self) -> &[Abi] { + &self.abis + } + pub fn schema_version(&self) -> u32 { + SCHEMA_VERSION + } + pub fn owned_bytes(&self) -> usize { + self.owned_bytes + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GpuBackend { + Metal, + Cuda, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DeviceKind { + Cpu, + Gpu(GpuBackend), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExecutionReality { + Cpu, + PhysicalGpu, + Simulation, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DeviceCapabilities { + pub device_id: String, + pub kind: DeviceKind, + pub reality: ExecutionReality, + pub target_abis: Vec, + pub max_batch_bytes: usize, + pub cpu_credit_cost: usize, +} + +impl DeviceCapabilities { + pub fn cpu(device_id: impl Into) -> Self { + Self { + device_id: device_id.into(), + kind: DeviceKind::Cpu, + reality: ExecutionReality::Cpu, + target_abis: vec![], + max_batch_bytes: usize::MAX, + cpu_credit_cost: 1, + } + } + /// Only an adapter that actually uses this physical device may advertise it. + /// This declaration is not GPU validation or multi-device scaling evidence. + pub fn physical_gpu(backend: GpuBackend, device_id: impl Into) -> Self { + Self { + kind: DeviceKind::Gpu(backend), + reality: ExecutionReality::PhysicalGpu, + ..Self::cpu(device_id) + } + } + pub fn simulated_gpu(backend: GpuBackend, device_id: impl Into) -> Self { + Self { + kind: DeviceKind::Gpu(backend), + reality: ExecutionReality::Simulation, + ..Self::cpu(device_id) + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ExecutionRecord { + Cpu, + Simulated { + backend: GpuBackend, + device_id: String, + }, + PhysicalGpu { + backend: GpuBackend, + device_id: String, + kernel_dispatches: u64, + kernel_output_used: bool, + }, +} +impl ExecutionRecord { + pub fn is_physical_gpu(&self) -> bool { + matches!(self, Self::PhysicalGpu { kernel_dispatches, kernel_output_used: true, .. } if *kernel_dispatches > 0) + } +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum RelocationKind { + Absolute64, + Relative32, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Relocation { + pub offset: u32, + pub target: u32, + pub kind: RelocationKind, + pub addend: i64, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FunctionEmission { + pub symbol: u32, + pub code: Vec, + pub relocations: Vec, + pub unwind: Vec, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Emission { + pub functions: Vec, + pub execution: ExecutionRecord, +} + +/// A submitted execution owns all storage through completion. `wait` must not +/// return until kernels/copies are complete or device teardown proves quiescence. +/// Dropping this object, including during unwinding, must also preserve that +/// guarantee. Cancellation does not invoke Drop early or reclaim its buffers. +pub trait PendingExecution: Send { + fn wait(self: Box) -> Result; +} + +pub trait BatchExecutor: Send { + fn capabilities(&self) -> DeviceCapabilities; + fn prepare(&mut self, _batch: &FlatBatch) -> Result<(), String> { + Ok(()) + } + /// An error means no submitted work still uses the batch. Return a pending + /// handle instead when recovery must await outstanding device work. + fn submit(&mut self, batch: Arc) -> Result, String>; +} + +type Callback = dyn Fn(&FlatBatch) -> Result + Send + Sync; +pub struct CpuCallbackExecutor { + capabilities: DeviceCapabilities, + callback: Arc, +} +impl CpuCallbackExecutor { + pub fn new( + device_id: impl Into, + callback: impl Fn(&FlatBatch) -> Result + Send + Sync + 'static, + ) -> Self { + Self { + capabilities: DeviceCapabilities::cpu(device_id), + callback: Arc::new(callback), + } + } +} +struct CallbackPending { + batch: Arc, + callback: Arc, +} +impl PendingExecution for CallbackPending { + fn wait(self: Box) -> Result { + (self.callback)(&self.batch) + } +} +impl BatchExecutor for CpuCallbackExecutor { + fn capabilities(&self) -> DeviceCapabilities { + self.capabilities.clone() + } + fn submit(&mut self, batch: Arc) -> Result, String> { + Ok(Box::new(CallbackPending { + batch, + callback: self.callback.clone(), + })) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Placement { + Automatic, + Cpu, + RequireGpu(GpuBackend), + SimulatedGpu(GpuBackend), +} + +#[derive(Clone, Debug)] +pub struct SubmitOptions { + pub placement: Placement, + pub prerequisites: Vec, + pub critical_path_cost: u64, + pub estimated_work: u64, +} +impl Default for SubmitOptions { + fn default() -> Self { + Self { + placement: Placement::Automatic, + prerequisites: vec![], + critical_path_cost: 0, + estimated_work: 0, + } + } +} + +#[derive(Clone, Debug)] +pub struct SessionConfig { + pub max_queued_batches: usize, + /// Bounds service-owned input storage, including pending/canceled work. + /// Device scratch/output budgets belong to the executor's capability policy. + pub max_owned_bytes: usize, + pub starvation_after_dispatches: u64, + pub cpu_credit_limit: usize, +} +impl Default for SessionConfig { + fn default() -> Self { + Self { + max_queued_batches: 32, + max_owned_bytes: 128 * 1024 * 1024, + starvation_after_dispatches: 16, + cpu_credit_limit: thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ServiceError { + Invalid(String), + Unavailable(String), + Backpressure, + UnknownClient, + Duplicate, + Stale, + UnknownPrerequisite, + DependencyFailed, + Canceled, + Shutdown, + Execution(String), + Output(String), + WorkerPanicked, +} +impl fmt::Display for ServiceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self:?}") + } +} +impl std::error::Error for ServiceError {} + +#[derive(Clone, Debug)] +pub struct RouteDecision { + pub device_id: String, + pub kind: DeviceKind, + pub reality: ExecutionReality, + pub reason: String, +} +#[derive(Clone, Debug, Default)] +pub struct StageCostLedger { + pub validation: Duration, + pub queue: Duration, + pub prepare: Duration, + pub submit: Duration, + pub wait: Duration, + pub assemble: Duration, +} +#[derive(Clone, Debug)] +pub struct CompletionReceipt { + pub session_id: u64, + pub batch_key: BatchKey, + pub schema_version: u32, + pub identity: BatchIdentity, + pub route: RouteDecision, + pub execution: ExecutionRecord, + pub output: Emission, + pub costs: StageCostLedger, +} + +#[derive(Clone, Debug)] +pub struct Client { + session: u64, + id: String, + registration: u64, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum JobStatus { + Queued, + Running, + CancellationPending, + Complete, +} +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Cancellation { + pub was_running: bool, + pub resources_pending: bool, +} +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct QueueSnapshot { + pub queued: usize, + pub running: usize, + pub owned_bytes: usize, + pub dispatches: u64, +} + +enum JobPhase { + Queued, + Running, + Done(Result, ServiceError>), + Retired(Result<(), ServiceError>), +} +struct Job { + batch: Option>, + client: String, + options: SubmitOptions, + eligible_devices: Vec, + phase: JobPhase, + cancel: bool, + enqueue_time: Instant, + enqueue_dispatch: u64, + sequence: u64, + bytes: usize, +} +struct Credit { + total: usize, + free: usize, + registration: u64, +} +struct State { + session: u64, + config: SessionConfig, + devices: Vec, + clients: BTreeMap, + jobs: BTreeMap, + latest: HashMap, + queued: usize, + owned_bytes: usize, + dispatches: u64, + sequence: u64, + next_client: u64, + shutdown: bool, +} +struct Shared { + state: Mutex, + changed: Condvar, +} +pub struct BuildSession { + shared: Arc, + workers: Vec>, +} +#[derive(Clone)] +pub struct Ticket { + key: BatchKey, + shared: Arc, + _ownership: Arc, +} +struct TicketOwnership { + key: BatchKey, + shared: Arc, +} + +impl BuildSession { + pub fn new( + config: SessionConfig, + executors: Vec>, + ) -> Result { + if config.max_queued_batches == 0 + || config.max_owned_bytes == 0 + || config.cpu_credit_limit == 0 + || config.starvation_after_dispatches == 0 + || executors.is_empty() + { + return Err(ServiceError::Invalid( + "positive budgets and at least one executor are required".into(), + )); + } + let devices: Vec<_> = executors.iter().map(|e| e.capabilities()).collect(); + let mut ids = HashSet::new(); + for device in &devices { + let coherent = matches!( + (device.kind, device.reality), + (DeviceKind::Cpu, ExecutionReality::Cpu) + | ( + DeviceKind::Gpu(_), + ExecutionReality::PhysicalGpu | ExecutionReality::Simulation + ) + ); + if !coherent + || device.device_id.is_empty() + || !ids.insert(device.device_id.clone()) + || device.cpu_credit_cost == 0 + || device.cpu_credit_cost > config.cpu_credit_limit + || device.max_batch_bytes == 0 + { + return Err(ServiceError::Invalid( + "inconsistent or duplicate device capability record".into(), + )); + } + } + let shared = Arc::new(Shared { + state: Mutex::new(State { + session: NEXT_SESSION.fetch_add(1, Ordering::Relaxed), + config, + devices: devices.clone(), + clients: BTreeMap::new(), + jobs: BTreeMap::new(), + latest: HashMap::new(), + queued: 0, + owned_bytes: 0, + dispatches: 0, + sequence: 0, + next_client: 0, + shutdown: false, + }), + changed: Condvar::new(), + }); + let workers = executors + .into_iter() + .zip(devices) + .map(|(executor, device)| { + let shared = shared.clone(); + thread::spawn(move || worker(shared, device, executor)) + }) + .collect(); + Ok(Self { shared, workers }) + } + pub fn capabilities(&self) -> Vec { + self.shared.state.lock().unwrap().devices.clone() + } + /// Checks lineage at this instant. The consumer must coordinate its final + /// external commit with source invalidation; this is not a file transaction. + pub fn is_current(&self, receipt: &CompletionReceipt) -> bool { + let state = self.shared.state.lock().unwrap(); + receipt.session_id == state.session + && state.latest.get(&receipt.batch_key.logical_id) + == Some(&receipt.batch_key.generation) + && state.jobs.get(&receipt.batch_key).is_some_and(|job| { + matches!(job.phase, JobPhase::Done(Ok(_)) | JobPhase::Retired(Ok(()))) + }) + } + /// Caller supplies credits already reserved for this build. They are held + /// while an executor runs or awaits a device; no extra CPU threads are lent + /// without an explicit credit. Registration does not claim jobserver access. + pub fn register_client( + &self, + id: impl Into, + cpu_credits: usize, + ) -> Result { + let id = id.into(); + let mut state = self.shared.state.lock().unwrap(); + if state.shutdown { + return Err(ServiceError::Shutdown); + } + let assigned: usize = state.clients.values().map(|c| c.total).sum(); + if id.is_empty() + || state.clients.contains_key(&id) + || cpu_credits == 0 + || cpu_credits > state.config.cpu_credit_limit.saturating_sub(assigned) + { + return Err(ServiceError::Invalid( + "client identity or reserved CPU-credit budget invalid".into(), + )); + } + let registration = state.next_client; + state.next_client += 1; + state.clients.insert( + id.clone(), + Credit { + total: cpu_credits, + free: cpu_credits, + registration, + }, + ); + Ok(Client { + session: state.session, + id, + registration, + }) + } + pub fn client_credits(&self, client: &Client) -> Result<(usize, usize), ServiceError> { + let state = self.shared.state.lock().unwrap(); + if state.session != client.session { + return Err(ServiceError::UnknownClient); + } + state + .clients + .get(&client.id) + .filter(|c| c.registration == client.registration) + .map(|c| (c.total, c.free)) + .ok_or(ServiceError::UnknownClient) + } + pub fn unregister_client(&self, client: &Client) -> Result<(), ServiceError> { + let mut state = self.shared.state.lock().unwrap(); + if state.session != client.session + || state + .clients + .get(&client.id) + .is_none_or(|c| c.registration != client.registration) + { + return Err(ServiceError::UnknownClient); + } + if state.jobs.values().any(|job| { + job.client == client.id && matches!(job.phase, JobPhase::Queued | JobPhase::Running) + }) { + return Err(ServiceError::Invalid( + "client still owns queued or pending work".into(), + )); + } + state.clients.remove(&client.id); + Ok(()) + } + pub fn snapshot(&self) -> QueueSnapshot { + let state = self.shared.state.lock().unwrap(); + QueueSnapshot { + queued: state.queued, + running: state + .jobs + .values() + .filter(|j| matches!(j.phase, JobPhase::Running)) + .count(), + owned_bytes: state.owned_bytes, + dispatches: state.dispatches, + } + } + pub fn submit( + &self, + client: &Client, + batch: Arc, + options: SubmitOptions, + ) -> Result { + let mut state = self.shared.state.lock().unwrap(); + if state.shutdown { + return Err(ServiceError::Shutdown); + } + let credit = state + .clients + .get(&client.id) + .filter(|credit| { + client.session == state.session && credit.registration == client.registration + }) + .ok_or(ServiceError::UnknownClient)?; + let eligible_devices: Vec<_> = state + .devices + .iter() + .filter(|d| { + eligible(d, &options.placement) + && d.cpu_credit_cost <= credit.total + && batch.owned_bytes() <= d.max_batch_bytes + && (d.target_abis.is_empty() + || d.target_abis.contains(&batch.identity().target_abi)) + }) + .map(|d| d.device_id.clone()) + .collect(); + if eligible_devices.is_empty() { + return Err(ServiceError::Unavailable( + "no eligible executor for placement, target, input size, and CPU credit".into(), + )); + } + if state.jobs.contains_key(batch.key()) { + return Err(ServiceError::Duplicate); + } + if state + .latest + .get(&batch.key().logical_id) + .is_some_and(|&generation| generation >= batch.key().generation) + { + return Err(ServiceError::Stale); + } + let mut prerequisites = HashSet::new(); + for key in &options.prerequisites { + if key == batch.key() || !prerequisites.insert(key) || !state.jobs.contains_key(key) { + return Err(ServiceError::UnknownPrerequisite); + } + if dependency_failed(state.jobs.get(key).unwrap()) { + return Err(ServiceError::DependencyFailed); + } + } + if state.queued >= state.config.max_queued_batches + || batch.owned_bytes() + > state + .config + .max_owned_bytes + .saturating_sub(state.owned_bytes) + { + return Err(ServiceError::Backpressure); + } + // Revisions supersede outputs from older queued and running requests. + // Running requests retain their input until their completion fence. + let superseded: Vec<_> = state + .jobs + .keys() + .filter(|key| key.logical_id == batch.key().logical_id) + .cloned() + .collect(); + for key in superseded { + cancel_job(&mut state, &key); + } + let key = batch.key().clone(); + let bytes = batch.owned_bytes(); + let enqueue_dispatch = state.dispatches; + let sequence = state.sequence; + state.sequence += 1; + state.latest.insert(key.logical_id.clone(), key.generation); + state.owned_bytes += bytes; + state.queued += 1; + state.jobs.insert( + key.clone(), + Job { + batch: Some(batch), + client: client.id.clone(), + options, + eligible_devices, + phase: JobPhase::Queued, + cancel: false, + enqueue_time: Instant::now(), + enqueue_dispatch, + sequence, + bytes, + }, + ); + self.shared.changed.notify_all(); + Ok(Ticket { + key: key.clone(), + shared: self.shared.clone(), + _ownership: Arc::new(TicketOwnership { + key, + shared: self.shared.clone(), + }), + }) + } +} + +impl Drop for BuildSession { + fn drop(&mut self) { + { + let mut state = self.shared.state.lock().unwrap(); + state.shutdown = true; + let keys: Vec<_> = state.jobs.keys().cloned().collect(); + for key in keys { + cancel_job(&mut state, &key); + } + self.shared.changed.notify_all(); + } + // Join waits for real completion; cancellation cannot free device-owned + // memory merely because the build-scoped service is being destroyed. + for worker in self.workers.drain(..) { + let _ = worker.join(); + } + } +} + +impl Ticket { + pub fn key(&self) -> &BatchKey { + &self.key + } + pub fn status(&self) -> JobStatus { + let state = self.shared.state.lock().unwrap(); + let job = state.jobs.get(&self.key).unwrap(); + match job.phase { + JobPhase::Queued => JobStatus::Queued, + JobPhase::Running if job.cancel => JobStatus::CancellationPending, + JobPhase::Running => JobStatus::Running, + JobPhase::Done(_) | JobPhase::Retired(_) => JobStatus::Complete, + } + } + pub fn cancel(&self) -> Cancellation { + let mut state = self.shared.state.lock().unwrap(); + let result = cancel_job(&mut state, &self.key); + self.shared.changed.notify_all(); + result + } + /// Returns only after ownership can be released, even when canceled. + pub fn wait(&self) -> Result, ServiceError> { + let mut state = self.shared.state.lock().unwrap(); + loop { + if let JobPhase::Done(result) = &state.jobs.get(&self.key).unwrap().phase { + return result.clone(); + } + state = self.shared.changed.wait(state).unwrap(); + } + } +} + +impl Drop for TicketOwnership { + fn drop(&mut self) { + let mut state = self.shared.state.lock().unwrap(); + let job = state.jobs.get_mut(&self.key).unwrap(); + if let JobPhase::Done(result) = &job.phase { + // Keep only the prerequisite outcome. Outputs remain alive while a + // caller holds its receipt, without accumulating inside the session. + job.phase = JobPhase::Retired(result.as_ref().map(|_| ()).map_err(Clone::clone)); + } else { + cancel_job(&mut state, &self.key); + } + self.shared.changed.notify_all(); + } +} + +fn eligible(device: &DeviceCapabilities, placement: &Placement) -> bool { + match placement { + Placement::Automatic | Placement::Cpu => { + device.kind == DeviceKind::Cpu && device.reality == ExecutionReality::Cpu + } + Placement::RequireGpu(backend) => { + device.kind == DeviceKind::Gpu(*backend) + && device.reality == ExecutionReality::PhysicalGpu + } + Placement::SimulatedGpu(backend) => { + device.kind == DeviceKind::Gpu(*backend) + && device.reality == ExecutionReality::Simulation + } + } +} + +fn dependency_failed(job: &Job) -> bool { + job.cancel + || matches!( + job.phase, + JobPhase::Done(Err(_)) | JobPhase::Retired(Err(_)) + ) +} +fn cancel_job(state: &mut State, key: &BatchKey) -> Cancellation { + let job = state.jobs.get_mut(key).unwrap(); + let running = matches!(job.phase, JobPhase::Running); + if matches!(job.phase, JobPhase::Queued) { + job.cancel = true; + job.phase = JobPhase::Done(Err(ServiceError::Canceled)); + job.batch.take(); + state.queued -= 1; + state.owned_bytes -= job.bytes; + } else if running { + job.cancel = true; + } + Cancellation { + was_running: running, + resources_pending: running, + } +} + +fn finish_failed_dependencies(state: &mut State) { + loop { + let failed: Vec<_> = state + .jobs + .iter() + .filter(|(_, job)| { + matches!(job.phase, JobPhase::Queued) + && job + .options + .prerequisites + .iter() + .any(|key| dependency_failed(state.jobs.get(key).unwrap())) + }) + .map(|(key, _)| key.clone()) + .collect(); + if failed.is_empty() { + return; + } + for key in failed { + let job = state.jobs.get_mut(&key).unwrap(); + job.phase = JobPhase::Done(Err(ServiceError::DependencyFailed)); + job.batch.take(); + state.queued -= 1; + state.owned_bytes -= job.bytes; + } + } +} + +fn select_job(state: &State, device: &DeviceCapabilities) -> Option { + let eligible: Vec<_> = state + .jobs + .iter() + .filter(|(_, job)| { + matches!(job.phase, JobPhase::Queued) + && job.eligible_devices.contains(&device.device_id) + && state.clients[&job.client].free >= device.cpu_credit_cost + && job.options.prerequisites.iter().all(|key| { + matches!( + state.jobs[key].phase, + JobPhase::Done(Ok(_)) | JobPhase::Retired(Ok(())) + ) + }) + }) + .collect(); + let starved = eligible + .iter() + .filter(|(_, job)| { + state.dispatches.saturating_sub(job.enqueue_dispatch) + >= state.config.starvation_after_dispatches + }) + .min_by_key(|(_, job)| job.sequence); + if let Some((key, _)) = starved { + return Some((*key).clone()); + } + eligible + .into_iter() + .max_by_key(|(_, job)| { + ( + job.options.critical_path_cost, + job.options.estimated_work, + std::cmp::Reverse(job.sequence), + ) + }) + .map(|(key, _)| key.clone()) +} + +fn worker(shared: Arc, device: DeviceCapabilities, mut executor: Box) { + loop { + let (key, batch, queue_cost, client) = { + let mut state = shared.state.lock().unwrap(); + loop { + finish_failed_dependencies(&mut state); + shared.changed.notify_all(); + if state.shutdown { + return; + } + if let Some(key) = select_job(&state, &device) { + let job = state.jobs.get_mut(&key).unwrap(); + job.phase = JobPhase::Running; + let batch = job.batch.as_ref().unwrap().clone(); + let queue_cost = job.enqueue_time.elapsed(); + let client = job.client.clone(); + state.clients.get_mut(&client).unwrap().free -= device.cpu_credit_cost; + state.queued -= 1; + state.dispatches += 1; + shared.changed.notify_all(); + break (key, batch, queue_cost, client); + } + state = shared.changed.wait(state).unwrap(); + } + }; + let mut costs = StageCostLedger { + validation: batch.validation, + queue: queue_cost, + ..Default::default() + }; + let result = catch_unwind(AssertUnwindSafe(|| { + execute(executor.as_mut(), &device, batch.clone(), &mut costs) + })); + let result = result.unwrap_or(Err(ServiceError::WorkerPanicked)); + let identity = batch.identity().clone(); + let schema_version = batch.schema_version(); + // The service's worker ownership is dropped before returning credits. + drop(batch); + let mut state = shared.state.lock().unwrap(); + let session_id = state.session; + let job = state.jobs.get_mut(&key).unwrap(); + let reason = match job.options.placement { + Placement::Automatic => "uncalibrated automatic policy selects CPU", + Placement::Cpu => "CPU explicitly selected", + Placement::RequireGpu(_) => "physical GPU explicitly required", + Placement::SimulatedGpu(_) => "simulation explicitly selected; not GPU evidence", + }; + let result = if job.cancel { + Err(ServiceError::Canceled) + } else { + result.map(|output| { + Arc::new(CompletionReceipt { + session_id, + batch_key: key.clone(), + schema_version, + identity, + route: RouteDecision { + device_id: device.device_id.clone(), + kind: device.kind, + reality: device.reality, + reason: reason.into(), + }, + execution: output.execution.clone(), + output, + costs, + }) + }) + }; + job.phase = JobPhase::Done(result); + job.batch.take(); + state.owned_bytes -= job.bytes; + state.clients.get_mut(&client).unwrap().free += device.cpu_credit_cost; + finish_failed_dependencies(&mut state); + shared.changed.notify_all(); + } +} + +fn execute( + executor: &mut dyn BatchExecutor, + device: &DeviceCapabilities, + batch: Arc, + costs: &mut StageCostLedger, +) -> Result { + let start = Instant::now(); + let prepared = executor.prepare(&batch); + costs.prepare = start.elapsed(); + prepared.map_err(ServiceError::Execution)?; + let start = Instant::now(); + let submitted = executor.submit(batch.clone()); + costs.submit = start.elapsed(); + let pending = submitted.map_err(ServiceError::Execution)?; + let start = Instant::now(); + let waited = pending.wait(); + costs.wait = start.elapsed(); + let mut output = waited.map_err(ServiceError::Execution)?; + let start = Instant::now(); + let result = validate_output(&batch, device, &mut output); + costs.assemble = start.elapsed(); + result?; + Ok(output) +} + +fn validate_output( + batch: &FlatBatch, + device: &DeviceCapabilities, + output: &mut Emission, +) -> Result<(), ServiceError> { + let attribution_valid = match (&output.execution, device.kind, device.reality) { + (ExecutionRecord::Cpu, DeviceKind::Cpu, ExecutionReality::Cpu) => true, + ( + ExecutionRecord::Simulated { backend, device_id }, + DeviceKind::Gpu(expected), + ExecutionReality::Simulation, + ) => *backend == expected && *device_id == device.device_id, + ( + ExecutionRecord::PhysicalGpu { + backend, + device_id, + kernel_dispatches, + kernel_output_used, + }, + DeviceKind::Gpu(expected), + ExecutionReality::PhysicalGpu, + ) => { + *backend == expected + && *device_id == device.device_id + && *kernel_dispatches > 0 + && *kernel_output_used + } + _ => false, + }; + if !attribution_valid { + return Err(ServiceError::Output( + "execution record contradicts registered device or contains no used kernel output" + .into(), + )); + } + let mut symbols = HashSet::new(); + if output.functions.len() != batch.functions().len() { + return Err(ServiceError::Output("partial function output".into())); + } + let mut inputs = vec![None; batch.symbols().len()]; + for function in batch.functions() { + inputs[function.symbol as usize] = Some(function); + } + for function in &mut output.functions { + let input = inputs + .get(function.symbol as usize) + .and_then(|f| *f) + .ok_or_else(|| ServiceError::Output("output symbol is absent from batch".into()))?; + if !symbols.insert(function.symbol) || function.code.is_empty() { + return Err(ServiceError::Output( + "empty code or duplicate function output".into(), + )); + } + if batch.abis()[input.abi as usize].requires_unwind && function.unwind.is_empty() { + return Err(ServiceError::Output( + "required unwind record is missing".into(), + )); + } + function + .relocations + .sort_by_key(|r| (r.offset, r.kind, r.target, r.addend)); + let mut end = 0; + for relocation in &function.relocations { + let width = match relocation.kind { + RelocationKind::Absolute64 => 8, + RelocationKind::Relative32 => 4, + }; + let current_end = (relocation.offset as usize) + .checked_add(width) + .ok_or_else(|| ServiceError::Output("relocation range overflow".into()))?; + if relocation.target as usize >= batch.symbols().len() + || (relocation.offset as usize) < end + || current_end > function.code.len() + { + return Err(ServiceError::Output( + "invalid or overlapping relocation".into(), + )); + } + end = current_end; + } + } + output.functions.sort_by(|a, b| { + batch.symbols()[a.symbol as usize] + .name + .cmp(&batch.symbols()[b.symbol as usize].name) + }); + Ok(()) +} diff --git a/experiments/gpu-rust-compiler/compiler/src/cuda.rs b/experiments/gpu-rust-compiler/compiler/src/cuda.rs new file mode 100644 index 00000000000..b06e6d5b935 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/cuda.rs @@ -0,0 +1,665 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A dynamically loaded, persistent CUDA context. The CPU artifact can load a +//! library built on its GPU runner without linking to CUDA on the build host. + +use crate::analysis::{GPUGroup, PackedAnalysis}; +use std::path::Path; + +#[repr(u32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Algorithm { + Dense = 0, + Sparse = 1, +} + +/// Placement remains independent from the fixed-point algorithm. Capability +/// discovery alone does not authorize a coherent-memory execution strategy. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlacementPolicy { + PinnedStaging, + CoherentUnverified, +} + +#[repr(C)] +#[derive(Default, Clone, Copy, Debug)] +pub struct Stats { + pub initialization_ms: f64, + pub staging_ms: f64, + pub host_to_device_ms: f64, + pub gpu_ms: f64, + pub device_to_host_ms: f64, + pub total_ms: f64, + pub max_sweeps: u32, + pub reused_context: u32, + pub device_allocations: u32, + pub pinned_allocations: u32, + pub resident_input: u32, + pub frontier_visits: u64, + pub edges_examined: u64, + pub discovered_bits: u64, +} + +#[derive(Default, Clone, Debug)] +pub struct Capabilities { + pub device_name: String, + pub device_uuid: String, + pub placement: String, + pub driver_version: i32, + pub runtime_version: i32, + pub compute_major: i32, + pub compute_minor: i32, + pub unified_addressing: bool, + pub managed_memory: bool, + pub concurrent_managed_access: bool, + pub pageable_memory_access: bool, + pub host_page_tables: bool, + pub direct_managed_access: bool, + pub async_engine_count: i32, + pub coherent_placement_verified: bool, +} + +fn json_string(value: &str) -> String { + let mut out = String::from("\""); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c.is_control() => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} + +impl Stats { + pub fn json(&self) -> String { + format!(concat!("{{\"initialization_ms\":{},\"staging_ms\":{},\"host_to_device_ms\":{},", + "\"gpu_ms\":{},\"device_to_host_ms\":{},\"total_ms\":{},\"max_sweeps\":{},", + "\"reused_context\":{},\"device_allocations\":{},\"pinned_allocations\":{},", + "\"resident_input\":{},\"frontier_visits\":{},\"edges_examined\":{},\"discovered_bits\":{}}}"), + self.initialization_ms, self.staging_ms, self.host_to_device_ms, self.gpu_ms, + self.device_to_host_ms, self.total_ms, self.max_sweeps, self.reused_context, + self.device_allocations, self.pinned_allocations, self.resident_input, + self.frontier_visits, self.edges_examined, self.discovered_bits) + } +} +impl Capabilities { + pub fn fingerprint(&self, cpu_workers: usize) -> String { + let cpu = std::fs::read_to_string("/proc/cpuinfo") + .ok() + .and_then(|info| { + info.lines().find_map(|line| { + line.strip_prefix("model name") + .and_then(|name| name.split_once(':')) + .map(|(_, name)| name.trim().to_owned()) + }) + }) + .unwrap_or_else(|| std::env::consts::ARCH.to_owned()); + format!( + "cuda-v1:{}:driver{}:runtime{}:cc{}.{}:cpu{}:{}:pinned_pcie", + self.device_uuid, + self.driver_version, + self.runtime_version, + self.compute_major, + self.compute_minor, + cpu_workers, + cpu + ) + } + pub fn json(&self) -> String { + format!(concat!("{{\"device_name\":{},\"device_uuid\":{},\"placement\":{},", + "\"driver_version\":{},\"runtime_version\":{},\"compute_major\":{},\"compute_minor\":{},", + "\"unified_addressing\":{},\"managed_memory\":{},\"concurrent_managed_access\":{},", + "\"pageable_memory_access\":{},\"hmm_capability_candidate\":{},\"host_page_tables\":{},", + "\"direct_managed_access\":{},\"async_engine_count\":{},\"coherent_placement_verified\":{}}}"), + json_string(&self.device_name), json_string(&self.device_uuid), json_string(&self.placement), + self.driver_version, self.runtime_version, self.compute_major, self.compute_minor, + self.unified_addressing, self.managed_memory, self.concurrent_managed_access, + self.pageable_memory_access, self.pageable_memory_access && !self.host_page_tables, + self.host_page_tables, self.direct_managed_access, self.async_engine_count, self.coherent_placement_verified) + } +} + +fn validate_packed(packed: &PackedAnalysis) -> Result<(), String> { + if packed.row_offsets.len() < 2 + || packed.functions.is_empty() + || packed.groups.is_empty() + || packed.total_cells == 0 + || packed.total_cells > u32::MAX as usize + || packed.successor_offsets.len() != packed.row_offsets.len() + || packed.predecessor_offsets.len() != packed.row_offsets.len() + || packed.uses.len() != packed.total_cells + || packed.defs.len() != packed.total_cells + || packed.phi_out.len() != packed.total_cells + || packed.predecessors.len() != packed.successors.len() + { + return Err("Invalid CUDA slice dimensions".into()); + } + let blocks = packed.block_count(); + let edges = packed.successors.len(); + for offsets in [&packed.successor_offsets, &packed.predecessor_offsets] { + if offsets[0] != 0 + || offsets[blocks] as usize != edges + || offsets + .windows(2) + .any(|w| w[0] > w[1] || w[1] as usize > edges) + { + return Err("Invalid CUDA CSR offsets".into()); + } + } + let mut block_base = 0; + let mut row_base = 0; + let mut group_index = 0; + for function in &packed.functions { + if function.block_count == 0 + || function.words == 0 + || function.block_base != block_base + || function.row_base != row_base + { + return Err("CUDA functions must cover packed blocks and cells exactly once".into()); + } + let end = block_base + .checked_add(function.block_count) + .ok_or("CUDA block dimensions overflow")?; + if end > blocks { + return Err("CUDA function exceeds packed blocks".into()); + } + for word in 0..function.words { + let Some(group) = packed.groups.get(group_index) else { + return Err("Missing CUDA function-word group".into()); + }; + if group.block_base as usize != block_base + || group.row_base as usize != row_base + || group.block_count as usize != function.block_count + || group.words as usize != function.words + || group.word as usize != word + { + return Err("Inconsistent CUDA function-word group".into()); + } + group_index += 1; + } + for block in block_base..end { + if packed.row_offsets[block] != row_base + (block - block_base) * function.words { + return Err("Invalid CUDA bitset row offsets".into()); + } + for (&start, &finish, values) in [ + ( + &packed.successor_offsets[block], + &packed.successor_offsets[block + 1], + &packed.successors, + ), + ( + &packed.predecessor_offsets[block], + &packed.predecessor_offsets[block + 1], + &packed.predecessors, + ), + ] { + if values[start as usize..finish as usize] + .iter() + .any(|&target| (target as usize) < block_base || target as usize >= end) + { + return Err("CUDA CFG edge escapes its function".into()); + } + } + } + block_base = end; + row_base = row_base + .checked_add( + function + .block_count + .checked_mul(function.words) + .ok_or("CUDA cell dimensions overflow")?, + ) + .ok_or("CUDA cell dimensions overflow")?; + } + if block_base != blocks + || row_base != packed.total_cells + || packed.row_offsets[blocks] != row_base + || group_index != packed.groups.len() + { + return Err("Incomplete CUDA function-word coverage".into()); + } + // Both representations describe the same multiset, including repeated edges. + let mut forward = Vec::with_capacity(edges); + let mut reverse = Vec::with_capacity(edges); + for block in 0..blocks { + for edge in packed.successor_offsets[block]..packed.successor_offsets[block + 1] { + forward.push((block as u32, packed.successors[edge as usize])); + } + for edge in packed.predecessor_offsets[block]..packed.predecessor_offsets[block + 1] { + reverse.push((packed.predecessors[edge as usize], block as u32)); + } + } + forward.sort_unstable(); + reverse.sort_unstable(); + if forward != reverse { + return Err("CUDA predecessor CSR does not reverse successor CSR".into()); + } + Ok(()) +} + +#[cfg(unix)] +mod platform { + use super::*; + use std::{ + ffi::{c_char, c_int, c_void, CStr, CString}, + ptr::NonNull, + }; + + #[repr(C)] + struct Input { + function_count: u32, + block_count: u32, + cell_count: u32, + group_count: u32, + max_blocks: u32, + edge_count: u32, + groups: *const GPUGroup, + successor_offsets: *const u32, + successors: *const u32, + predecessor_offsets: *const u32, + predecessors: *const u32, + uses: *const u32, + defs: *const u32, + phi_out: *const u32, + } + + #[repr(C)] + struct NativeCapabilities { + device_name: [c_char; 256], + device_uuid: [c_char; 64], + driver_version: i32, + runtime_version: i32, + compute_major: i32, + compute_minor: i32, + unified_addressing: i32, + managed_memory: i32, + concurrent_managed_access: i32, + pageable_memory_access: i32, + host_page_tables: i32, + direct_managed_access: i32, + async_engine_count: i32, + coherent_placement_verified: i32, + } + + #[cfg_attr(target_os = "linux", link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const c_char, flags: c_int) -> *mut c_void; + fn dlsym(library: *mut c_void, name: *const c_char) -> *mut c_void; + fn dlerror() -> *const c_char; + fn dlclose(library: *mut c_void) -> c_int; + } + + type Create = unsafe extern "C" fn(*mut c_char, usize) -> *mut c_void; + type GetCapabilities = + unsafe extern "C" fn(*mut c_void, *mut NativeCapabilities, *mut c_char, usize) -> c_int; + type Submit = unsafe extern "C" fn(*mut c_void, *const Input, u32, *mut c_char, usize) -> c_int; + type SubmitResident = unsafe extern "C" fn(*mut c_void, u32, *mut c_char, usize) -> c_int; + type Pending = unsafe extern "C" fn(*mut c_void, *mut u32, *mut c_char, usize) -> c_int; + type Finish = + unsafe extern "C" fn(*mut c_void, *mut u32, usize, *mut Stats, *mut c_char, usize) -> c_int; + type Destroy = unsafe extern "C" fn(*mut c_void); + + fn message(bytes: &[c_char]) -> String { + let bytes: Vec<_> = bytes + .iter() + .take_while(|&&b| b != 0) + .map(|&b| b as u8) + .collect(); + String::from_utf8_lossy(&bytes).into_owned() + } + + fn loader_error() -> String { + let error = unsafe { dlerror() }; + if error.is_null() { + "CUDA library loader failed".into() + } else { + unsafe { CStr::from_ptr(error) } + .to_string_lossy() + .into_owned() + } + } + + struct Library(NonNull); + impl Library { + fn open(path: &Path) -> Result { + let path = CString::new(path.as_os_str().as_encoded_bytes()) + .map_err(|_| "CUDA library path contains NUL")?; + // RTLD_NOW resolves every dependency before any CUDA context exists. + NonNull::new(unsafe { dlopen(path.as_ptr(), 2) }) + .map(Self) + .ok_or_else(loader_error) + } + unsafe fn symbol(&self, name: &[u8]) -> Result { + assert_eq!(std::mem::size_of::(), std::mem::size_of::<*mut c_void>()); + unsafe { dlerror() }; + let pointer = unsafe { dlsym(self.0.as_ptr(), name.as_ptr().cast()) }; + if pointer.is_null() { + return Err(loader_error()); + } + Ok(unsafe { std::mem::transmute_copy(&pointer) }) + } + } + impl Drop for Library { + fn drop(&mut self) { + unsafe { dlclose(self.0.as_ptr()) }; + } + } + + pub struct Context { + pointer: NonNull, + submit_fn: Submit, + resident_fn: SubmitResident, + pending_fn: Pending, + finish_fn: Finish, + destroy_fn: Destroy, + capabilities: Capabilities, + in_flight_cells: Option, + resident_cells: Option, + validation_ms: f64, + pub submissions: usize, + _library: Library, + } + impl Context { + pub fn new_with_placement(path: &Path, placement: PlacementPolicy) -> Result { + if placement == PlacementPolicy::CoherentUnverified { + return Err("Coherent CUDA placement is unverified; use pinned staging until measured on coherent hardware".into()); + } + Self::load(path) + } + pub fn new(path: &Path) -> Result { + Self::new_with_placement(path, PlacementPolicy::PinnedStaging) + } + fn load(path: &Path) -> Result { + let library = Library::open(path)?; + let version: unsafe extern "C" fn() -> u32 = + unsafe { library.symbol(b"gpulab_cuda_abi_version\0")? }; + if unsafe { version() } != 1 { + return Err("CUDA library ABI version must be 1".into()); + } + let create: Create = unsafe { library.symbol(b"gpulab_cuda_create\0")? }; + let capabilities_fn: GetCapabilities = + unsafe { library.symbol(b"gpulab_cuda_capabilities\0")? }; + let submit_fn = unsafe { library.symbol(b"gpulab_cuda_submit\0")? }; + let resident_fn = unsafe { library.symbol(b"gpulab_cuda_submit_resident\0")? }; + let pending_fn = unsafe { library.symbol(b"gpulab_cuda_pending\0")? }; + let finish_fn = unsafe { library.symbol(b"gpulab_cuda_finish\0")? }; + let destroy_fn: Destroy = unsafe { library.symbol(b"gpulab_cuda_destroy\0")? }; + let mut error = [0 as c_char; 1024]; + let pointer = NonNull::new(unsafe { create(error.as_mut_ptr(), error.len()) }) + .ok_or_else(|| message(&error))?; + let mut native: NativeCapabilities = unsafe { std::mem::zeroed() }; + if unsafe { + capabilities_fn( + pointer.as_ptr(), + &mut native, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + unsafe { destroy_fn(pointer.as_ptr()) }; + return Err(message(&error)); + } + let capabilities = Capabilities { + device_name: message(&native.device_name), + device_uuid: message(&native.device_uuid), + placement: "pinned_pcie".into(), + driver_version: native.driver_version, + runtime_version: native.runtime_version, + compute_major: native.compute_major, + compute_minor: native.compute_minor, + unified_addressing: native.unified_addressing != 0, + managed_memory: native.managed_memory != 0, + concurrent_managed_access: native.concurrent_managed_access != 0, + pageable_memory_access: native.pageable_memory_access != 0, + host_page_tables: native.host_page_tables != 0, + direct_managed_access: native.direct_managed_access != 0, + async_engine_count: native.async_engine_count, + coherent_placement_verified: native.coherent_placement_verified != 0, + }; + Ok(Self { + pointer, + submit_fn, + resident_fn, + pending_fn, + finish_fn, + destroy_fn, + capabilities, + in_flight_cells: None, + resident_cells: None, + validation_ms: 0.0, + submissions: 0, + _library: library, + }) + } + + pub fn capabilities(&self) -> Capabilities { + self.capabilities.clone() + } + + pub fn submit( + &mut self, + packed: &PackedAnalysis, + algorithm: Algorithm, + ) -> Result<(), String> { + if self.in_flight_cells.is_some() { + return Err("Native CUDA context already has an in-flight pass".into()); + } + let validation_start = std::time::Instant::now(); + validate_packed(packed)?; + let count = |value: usize| { + u32::try_from(value).map_err(|_| "CUDA input exceeds u32 dimensions".to_string()) + }; + let input = Input { + function_count: count(packed.functions.len())?, + block_count: count(packed.block_count())?, + cell_count: count(packed.total_cells)?, + group_count: count(packed.groups.len())?, + max_blocks: count(packed.max_blocks())?, + edge_count: count(packed.successors.len())?, + groups: packed.groups.as_ptr(), + successor_offsets: packed.successor_offsets.as_ptr(), + successors: packed.successors.as_ptr(), + predecessor_offsets: packed.predecessor_offsets.as_ptr(), + predecessors: packed.predecessors.as_ptr(), + uses: packed.uses.as_ptr(), + defs: packed.defs.as_ptr(), + phi_out: packed.phi_out.as_ptr(), + }; + self.validation_ms = validation_start.elapsed().as_secs_f64() * 1000.0; + let mut error = [0 as c_char; 1024]; + if unsafe { + (self.submit_fn)( + self.pointer.as_ptr(), + &input, + algorithm as u32, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + self.resident_cells = None; + return Err(message(&error)); + } + self.in_flight_cells = Some(packed.total_cells); + self.resident_cells = Some(packed.total_cells); + self.submissions += 1; + Ok(()) + } + + /// Replay only the previous upload. This does not accept changed facts. + pub fn submit_resident(&mut self, algorithm: Algorithm) -> Result<(), String> { + if self.in_flight_cells.is_some() { + return Err("Native CUDA context already has an in-flight pass".into()); + } + let cells = self.resident_cells.ok_or("No CUDA input is resident")?; + self.validation_ms = 0.0; + let mut error = [0 as c_char; 1024]; + if unsafe { + (self.resident_fn)( + self.pointer.as_ptr(), + algorithm as u32, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + self.resident_cells = None; + return Err(message(&error)); + } + self.in_flight_cells = Some(cells); + self.submissions += 1; + Ok(()) + } + + /// Poll the completion event. This never waits or consumes a result. + pub fn is_pending(&mut self) -> Result { + let mut pending = 0; + let mut error = [0 as c_char; 1024]; + if unsafe { + (self.pending_fn)( + self.pointer.as_ptr(), + &mut pending, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + self.in_flight_cells = None; + self.resident_cells = None; + return Err(message(&error)); + } + Ok(pending != 0) + } + + pub fn finish(&mut self) -> Result<(Vec, Stats), String> { + let cells = self + .in_flight_cells + .take() + .ok_or("No in-flight CUDA pass")?; + let mut output = vec![0; cells]; + let mut stats = Stats::default(); + let mut error = [0 as c_char; 1024]; + if unsafe { + (self.finish_fn)( + self.pointer.as_ptr(), + output.as_mut_ptr(), + cells, + &mut stats, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + self.resident_cells = None; + return Err(message(&error)); + } + stats.staging_ms += self.validation_ms; + stats.total_ms += self.validation_ms; + Ok((output, stats)) + } + } + impl Drop for Context { + fn drop(&mut self) { + unsafe { (self.destroy_fn)(self.pointer.as_ptr()) }; + } + } +} + +#[cfg(not(unix))] +mod platform { + use super::*; + pub struct Context { + pub submissions: usize, + } + impl Context { + pub fn new(_: &Path) -> Result { + Err("Native CUDA library loading requires Unix; no GPU fallback was run".into()) + } + pub fn new_with_placement(path: &Path, placement: PlacementPolicy) -> Result { + if placement == PlacementPolicy::CoherentUnverified { + return Err("Coherent CUDA placement is unverified; use pinned staging until measured on coherent hardware".into()); + } + Self::new(path) + } + pub fn capabilities(&self) -> Capabilities { + Capabilities::default() + } + pub fn submit(&mut self, _: &PackedAnalysis, _: Algorithm) -> Result<(), String> { + Err("CUDA unavailable".into()) + } + pub fn submit_resident(&mut self, _: Algorithm) -> Result<(), String> { + Err("CUDA unavailable".into()) + } + pub fn finish(&mut self) -> Result<(Vec, Stats), String> { + Err("CUDA unavailable".into()) + } + pub fn is_pending(&mut self) -> Result { + Err("CUDA unavailable".into()) + } + } +} +pub use platform::Context; + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> PackedAnalysis { + PackedAnalysis { + functions: vec![crate::analysis::FunctionDesc { + module_index: 0, + block_base: 0, + block_count: 2, + row_base: 0, + words: 1, + value_count: 1, + }], + groups: vec![GPUGroup { + block_base: 0, + block_count: 2, + row_base: 0, + words: 1, + word: 0, + }], + successor_offsets: vec![0, 1, 1], + successors: vec![1], + predecessor_offsets: vec![0, 0, 1], + predecessors: vec![0], + row_offsets: vec![0, 1, 2], + uses: vec![0, 1], + defs: vec![0, 0], + phi_out: vec![0, 0], + total_cells: 2, + } + } + #[test] + fn malformed_csr_is_rejected_before_native_reads() { + let mut packed = fixture(); + packed.successor_offsets.pop(); + assert!(validate_packed(&packed).is_err()); + } + #[test] + fn a_missing_library_reports_a_loader_error() { + assert!(Context::new(Path::new("/no/such/gpulab-library.so")).is_err()); + } + #[test] + fn coherent_placement_is_rejected_until_hardware_is_verified() { + let error = Context::new_with_placement( + Path::new("/no/such/gpulab-library.so"), + PlacementPolicy::CoherentUnverified, + ) + .err() + .expect("unverified placement must fail"); + assert!( + error.contains("Coherent CUDA placement is unverified"), + "{error}" + ); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/executor.rs b/experiments/gpu-rust-compiler/compiler/src/executor.rs new file mode 100644 index 00000000000..af65c334713 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/executor.rs @@ -0,0 +1,677 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Persistent CPU workers for the existing sparse bitset worklist. +//! Inputs are immutable Arc snapshots. Every submission allocates zeroed facts; +//! buffer/thread reuse never carries liveness across changing source programs. + +use crate::analysis::{solve_cpu_serial, solve_words, PackedAnalysis}; +use std::{ + collections::{HashMap, HashSet}, + ops::Range, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, Arc, + }, + thread::{self, JoinHandle}, + time::Instant, +}; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum CpuMode { + Serial, + Functions, + Words, +} + +impl CpuMode { + pub const ALL: [Self; 3] = [Self::Serial, Self::Functions, Self::Words]; + pub fn name(self) -> &'static str { + match self { + Self::Serial => "serial", + Self::Functions => "function_pool", + Self::Words => "word_pool", + } + } +} + +#[derive(Clone, Debug)] +pub struct CpuTiming { + pub mode: CpuMode, + pub workers: usize, + pub samples_ms: Vec, + pub median_ms: f64, + pub p90_ms: f64, +} + +#[derive(Clone, Debug)] +pub struct CpuCalibration { + pub selected_mode: CpuMode, + pub selected_workers: usize, + pub variants: Vec, + pub repeats: usize, +} + +/// Indexed view of an already prepared pack; no CFG or GEN/DEF reconstruction. +#[derive(Clone, Debug)] +pub struct AnalysisSelection { + pub module_indices: Vec, + pub functions: Vec, + pub row_ranges: Vec>, + pub total_cells: usize, +} + +impl AnalysisSelection { + pub fn new(packed: &PackedAnalysis, module_indices: &[usize]) -> Result { + let mut seen = HashSet::new(); + let mut selection = Self { + module_indices: module_indices.to_vec(), + functions: Vec::new(), + row_ranges: Vec::new(), + total_cells: 0, + }; + for &module_index in module_indices { + if !seen.insert(module_index) { + return Err("Duplicate selected module function".into()); + } + let index = packed + .functions + .get(module_index) + .filter(|f| f.module_index == module_index) + .map(|_| module_index) + .or_else(|| { + packed + .functions + .iter() + .position(|f| f.module_index == module_index) + }) + .ok_or_else(|| format!("Selected module function {module_index} is absent"))?; + let f = &packed.functions[index]; + let count = f.block_count * f.words; + selection.functions.push(index); + selection.row_ranges.push(f.row_base..f.row_base + count); + selection.total_cells += count; + } + Ok(selection) + } + + fn complete(packed: &PackedAnalysis) -> Self { + Self { + module_indices: packed.functions.iter().map(|f| f.module_index).collect(), + functions: (0..packed.functions.len()).collect(), + row_ranges: packed + .functions + .iter() + .map(|f| f.row_base..f.row_base + f.block_count * f.words) + .collect(), + total_cells: packed.total_cells, + } + } +} + +#[derive(Clone, Debug)] +pub struct SubsetSolution { + pub values: Vec, + pub row_ranges: Vec>, +} + +impl SubsetSolution { + pub fn scatter_into(&self, output: &mut [u32]) -> Result<(), String> { + let mut source = 0; + for range in &self.row_ranges { + if range.end > output.len() || range.end < range.start { + return Err("Subset scatter destination dimensions differ".into()); + } + let end = source + range.len(); + if end > self.values.len() { + return Err("Subset scatter source dimensions differ".into()); + } + output[range.clone()].copy_from_slice(&self.values[source..end]); + source = end; + } + if source != self.values.len() { + return Err("Subset scatter has trailing facts".into()); + } + Ok(()) + } +} + +#[derive(Clone)] +struct Task { + function: usize, + first_word: usize, + words: usize, + output_base: usize, +} +struct Job { + packed: Arc, + tasks: Vec>, + next: AtomicUsize, +} +enum Command { + Solve(Arc), + Stop, +} +type WorkerOutput = Vec<(Task, Vec)>; + +/// Creation and destruction are outside warm solve timings. Scheduling, fresh +/// scratch/output allocation and result assembly remain inside every solve. +pub struct CpuExecutor { + workers: Vec<(mpsc::Sender, JoinHandle<()>)>, + completed: mpsc::Receiver, + selected: HashMap<[usize; 5], CpuMode>, + worker_limits: HashMap<([usize; 5], CpuMode), usize>, + jobs_completed: usize, + pool_submissions: usize, + initialization_ms: f64, +} + +impl CpuExecutor { + pub fn new(max_workers: usize) -> Self { + let initialization = Instant::now(); + let available = thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + let count = if max_workers == 0 { + available + } else { + available.min(max_workers) + } + .max(1); + let (completed_tx, completed) = mpsc::channel(); + let workers = (0..count) + .map(|_| { + let (sender, receiver) = mpsc::channel(); + let completed = completed_tx.clone(); + let handle = thread::spawn(move || { + while let Ok(Command::Solve(job)) = receiver.recv() { + let mut output = Vec::new(); + loop { + let task = job.next.fetch_add(1, Ordering::Relaxed); + let Some(batch) = job.tasks.get(task) else { + break; + }; + for task in batch { + let f = &job.packed.functions[task.function]; + output.push(( + task.clone(), + solve_words( + &job.packed, + f, + task.first_word..task.first_word + task.words, + ), + )); + } + } + if completed.send(output).is_err() { + break; + } + } + }); + (sender, handle) + }) + .collect(); + Self { + workers, + completed, + selected: HashMap::new(), + worker_limits: HashMap::new(), + jobs_completed: 0, + pool_submissions: 0, + initialization_ms: initialization.elapsed().as_secs_f64() * 1000.0, + } + } + + pub fn worker_count(&self) -> usize { + self.workers.len() + } + pub fn jobs_completed(&self) -> usize { + self.jobs_completed + } + pub fn workers(&self) -> usize { + self.worker_count() + } + pub fn requests(&self) -> usize { + self.jobs_completed() + } + pub fn pool_reused(&self) -> bool { + self.pool_submissions > 1 + } + pub fn pool_submissions(&self) -> usize { + self.pool_submissions + } + pub fn initialization_ms(&self) -> f64 { + self.initialization_ms + } + + fn key(packed: &PackedAnalysis) -> [usize; 5] { + [ + packed.functions.len(), + packed.block_count(), + packed.total_cells, + packed.max_blocks(), + packed.functions.iter().map(|f| f.words).max().unwrap_or(0), + ] + } + + /// Default to serial until CPU modes have been explicitly calibrated. A + /// resident pool is not an excuse to slow down tiny analyses with dispatch. + pub fn solve(&mut self, packed: Arc) -> Vec { + let mode = self + .selected + .get(&Self::key(&packed)) + .copied() + .unwrap_or(CpuMode::Serial); + self.solve_mode(packed, mode) + } + + pub fn solve_mode(&mut self, packed: Arc, mode: CpuMode) -> Vec { + let workers = self.worker_limit(&packed, mode); + self.solve_mode_with_workers(packed, mode, workers) + } + + pub fn worker_limit(&self, packed: &PackedAnalysis, mode: CpuMode) -> usize { + if mode == CpuMode::Serial { + 1 + } else { + self.worker_limits + .get(&(Self::key(packed), mode)) + .copied() + .unwrap_or(self.workers.len()) + } + } + + pub fn set_worker_limit( + &mut self, + packed: &PackedAnalysis, + mode: CpuMode, + workers: usize, + ) -> Result<(), String> { + if workers == 0 || workers > self.workers.len() { + return Err("CPU worker limit exceeds resident pool capacity".into()); + } + self.worker_limits + .insert((Self::key(packed), mode), workers); + Ok(()) + } + + pub fn solve_mode_with_workers( + &mut self, + packed: Arc, + mode: CpuMode, + workers: usize, + ) -> Vec { + if mode == CpuMode::Serial || workers <= 1 { + // Retain the exact optimized serial baseline, without adding + // indexed selection/task assembly overhead to a whole-pack solve. + self.jobs_completed += 1; + return solve_cpu_serial(&packed); + } + let selection = AnalysisSelection::complete(&packed); + self.solve_selection_with_workers(packed, &selection, mode, workers) + .values + } + + pub fn solve_selection( + &mut self, + packed: Arc, + selection: &AnalysisSelection, + mode: CpuMode, + ) -> SubsetSolution { + let workers = self.worker_limit(&packed, mode); + self.solve_selection_with_workers(packed, selection, mode, workers) + } + + pub fn solve_selection_with_workers( + &mut self, + packed: Arc, + selection: &AnalysisSelection, + mode: CpuMode, + workers: usize, + ) -> SubsetSolution { + let workers = workers.clamp(1, self.workers.len()); + if mode == CpuMode::Serial || workers == 1 { + let mut values = vec![0; selection.total_cells]; + let mut row = 0; + for &index in &selection.functions { + let f = &packed.functions[index]; + let solution = solve_words(&packed, f, 0..f.words); + values[row..row + solution.len()].copy_from_slice(&solution); + row += solution.len(); + } + self.jobs_completed += 1; + return SubsetSolution { + values, + row_ranges: selection.row_ranges.clone(), + }; + } + let mut tasks = Vec::new(); + let mut output_base = 0; + for &index in &selection.functions { + let f = &packed.functions[index]; + let width = if mode == CpuMode::Words { 4 } else { f.words }; + for first_word in (0..f.words).step_by(width) { + tasks.push(Task { + function: index, + first_word, + words: width.min(f.words - first_word), + output_base, + }); + } + output_base += f.block_count * f.words; + } + let mut result = vec![0; selection.total_cells]; + if tasks.len() <= 1 { + for task in tasks { + let f = &packed.functions[task.function]; + let values = solve_words(&packed, f, task.first_word..task.first_word + task.words); + assemble(&mut result, f.words, f.block_count, &task, &values); + } + } else { + // Amortize one atomic dequeue over several small functions while + // preserving dynamic balance for a few wide/long functions. + self.pool_submissions += 1; + let active_workers = workers.min(tasks.len()); + let batch_size = tasks.len().div_ceil(active_workers * 4).max(1); + let batches = tasks + .chunks(batch_size) + .map(|batch| batch.to_vec()) + .collect(); + let job = Arc::new(Job { + packed, + tasks: batches, + next: AtomicUsize::new(0), + }); + for (sender, _) in self.workers.iter().take(active_workers) { + sender + .send(Command::Solve(Arc::clone(&job))) + .expect("CPU worker stopped"); + } + for _ in 0..active_workers { + for (task, values) in self.completed.recv().expect("CPU worker did not complete") { + let f = &job.packed.functions[task.function]; + assemble(&mut result, f.words, f.block_count, &task, &values); + } + } + } + self.jobs_completed += 1; + SubsetSolution { + values: result, + row_ranges: selection.row_ranges.clone(), + } + } + + /// Calibration runs each mode interleaved and retains all timings. The + /// selected mode minimizes median elapsed cost, including pool dispatch. + pub fn calibrate( + &mut self, + packed: Arc, + repeats: usize, + ) -> Result { + if !(3..=10_000).contains(&repeats) { + return Err("CPU calibration requires 3..=10000 samples".into()); + } + let reference = self.solve_mode(Arc::clone(&packed), CpuMode::Serial); + let mut candidates = vec![(CpuMode::Serial, 1)]; + for mode in [CpuMode::Functions, CpuMode::Words] { + let tasks = if mode == CpuMode::Functions { + packed.functions.len() + } else { + packed.functions.iter().map(|f| f.words.div_ceil(4)).sum() + }; + let max_workers = self.workers.len().min(tasks).max(1); + let mut limits = Vec::new(); + let mut count = 2; + while count < max_workers { + limits.push(count); + count *= 2; + } + limits.push(max_workers); + candidates.extend(limits.into_iter().map(|workers| (mode, workers))); + } + let mut samples = vec![Vec::new(); candidates.len()]; + for iteration in 0..repeats { + for offset in 0..candidates.len() { + let index = (iteration + offset) % candidates.len(); + let start = Instant::now(); + let actual = self.solve_mode_with_workers( + Arc::clone(&packed), + candidates[index].0, + candidates[index].1, + ); + samples[index].push(start.elapsed().as_secs_f64() * 1000.0); + if actual != reference { + return Err("Persistent CPU modes disagree".into()); + } + } + } + let variants: Vec<_> = samples + .into_iter() + .enumerate() + .map(|(index, samples_ms)| { + let mut sorted = samples_ms.clone(); + sorted.sort_unstable_by(f64::total_cmp); + let median_ms = percentile(&sorted, 0.5); + let p90_ms = percentile(&sorted, 0.9); + CpuTiming { + mode: candidates[index].0, + workers: candidates[index].1, + samples_ms, + median_ms, + p90_ms, + } + }) + .collect(); + for mode in CpuMode::ALL { + let best = variants + .iter() + .filter(|v| v.mode == mode) + .min_by(|a, b| a.median_ms.total_cmp(&b.median_ms)) + .unwrap(); + self.worker_limits + .insert((Self::key(&packed), mode), best.workers); + } + let best = variants + .iter() + .min_by(|a, b| a.median_ms.total_cmp(&b.median_ms)) + .unwrap(); + let selected_mode = best.mode; + let selected_workers = best.workers; + self.selected.insert(Self::key(&packed), selected_mode); + Ok(CpuCalibration { + selected_mode, + selected_workers, + variants, + repeats, + }) + } +} + +fn percentile(sorted: &[f64], fraction: f64) -> f64 { + let position = (sorted.len() - 1) as f64 * fraction; + let first = position.floor() as usize; + let last = position.ceil() as usize; + sorted[first] + (sorted[last] - sorted[first]) * (position - first as f64) +} + +fn assemble(result: &mut [u32], width: usize, blocks: usize, task: &Task, values: &[u32]) { + if task.first_word == 0 && task.words == width { + result[task.output_base..task.output_base + values.len()].copy_from_slice(values); + } else { + for block in 0..blocks { + let target = task.output_base + block * width + task.first_word; + result[target..target + task.words] + .copy_from_slice(&values[block * task.words..(block + 1) * task.words]); + } + } +} + +impl Drop for CpuExecutor { + fn drop(&mut self) { + for (sender, _) in &self.workers { + let _ = sender.send(Command::Stop); + } + for (_, handle) in self.workers.drain(..) { + let _ = handle.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::{solve_cpu_serial, FunctionDesc, GPUGroup}; + fn pack(functions: usize, blocks: usize, words: usize, seed: u32) -> Arc { + let cells = functions * blocks * words; + let mut p = PackedAnalysis { + functions: Vec::new(), + groups: Vec::new(), + successor_offsets: vec![0], + successors: Vec::new(), + predecessor_offsets: vec![0], + predecessors: Vec::new(), + row_offsets: Vec::new(), + uses: vec![0; cells], + defs: vec![0; cells], + phi_out: vec![0; cells], + total_cells: cells, + }; + for f in 0..functions { + p.functions.push(FunctionDesc { + module_index: f * 2, + block_base: f * blocks, + block_count: blocks, + row_base: f * blocks * words, + words, + value_count: words * 32, + }); + for word in 0..words { + p.groups.push(GPUGroup { + block_base: (f * blocks) as u32, + block_count: blocks as u32, + row_base: (f * blocks * words) as u32, + words: words as u32, + word: word as u32, + }); + } + for b in 0..blocks { + p.row_offsets.push((f * blocks + b) * words); + if b + 1 < blocks { + p.successors.push((f * blocks + b + 1) as u32); + } + p.successor_offsets.push(p.successors.len() as u32); + if b > 0 { + p.predecessors.push((f * blocks + b - 1) as u32); + } + p.predecessor_offsets.push(p.predecessors.len() as u32); + for w in 0..words { + p.uses[((f * blocks + b) * words) + w] = + seed.rotate_left(((f + b + w) % 32) as u32); + } + } + } + p.row_offsets.push(cells); + Arc::new(p) + } + + #[test] + fn persistent_pool_matches_sparse_oracle_for_changing_and_resized_programs() { + let mut executor = CpuExecutor::new(4); + let workers = executor.worker_count(); + for input in [ + pack(37, 9, 7, 1), + pack(4, 257, 17, 2), + pack(37, 9, 7, 0), + pack(2, 3, 1, 8), + pack(0, 0, 0, 0), + ] { + let expected = solve_cpu_serial(&input); + for mode in CpuMode::ALL { + assert_eq!(executor.solve_mode(Arc::clone(&input), mode), expected); + } + assert_eq!(executor.worker_count(), workers); + } + assert_eq!(executor.jobs_completed(), 15); + if workers > 1 { + assert!(executor.pool_reused()); + assert!(executor.pool_submissions() > 1); + } + } + + #[test] + fn indexed_subset_preserves_module_ids_and_scatter_does_not_overwrite_neighbors() { + let input = pack(5, 7, 11, 7); + let selection = AnalysisSelection::new(&input, &[8, 2]).unwrap(); + assert_eq!(selection.functions, vec![4, 1]); + let mut executor = CpuExecutor::new(3); + let expected = solve_cpu_serial(&input); + for mode in CpuMode::ALL { + let subset = executor.solve_selection(Arc::clone(&input), &selection, mode); + let mut output = vec![u32::MAX; input.total_cells]; + subset.scatter_into(&mut output).unwrap(); + for f in &input.functions { + let range = f.row_base..f.row_base + f.block_count * f.words; + if selection.module_indices.contains(&f.module_index) { + assert_eq!(output[range.clone()], expected[range]); + } else { + assert!(output[range].iter().all(|v| *v == u32::MAX)); + } + } + } + assert!(AnalysisSelection::new(&input, &[2, 2]).is_err()); + assert!(AnalysisSelection::new(&input, &[1]).is_err()); + let subset = crate::analysis::select_packed(&input, &[8, 2]).unwrap(); + assert_eq!( + subset + .functions + .iter() + .map(|f| f.module_index) + .collect::>(), + vec![8, 2] + ); + assert_eq!( + solve_cpu_serial(&subset), + executor + .solve_selection(Arc::clone(&input), &selection, CpuMode::Serial) + .values + ); + let mut merged = vec![u32::MAX; input.total_cells]; + crate::analysis::scatter(&subset, &solve_cpu_serial(&subset), &input, &mut merged).unwrap(); + for range in &selection.row_ranges { + assert_eq!(merged[range.clone()], expected[range.clone()]); + } + assert!(crate::analysis::select_packed(&input, &[8, 8]).is_err()); + assert!(crate::analysis::select_packed(&input, &[1]).is_err()); + assert_eq!( + crate::analysis::select_packed(&input, &[]) + .unwrap() + .total_cells, + 0 + ); + } + + #[test] + fn measured_cpu_calibration_records_all_modes_and_reuses_pool() { + let input = pack(40, 7, 5, 3); + let mut executor = CpuExecutor::new(3); + assert!(executor.calibrate(Arc::clone(&input), 2).is_err()); + let calibration = executor.calibrate(Arc::clone(&input), 3).unwrap(); + assert!(calibration.variants.len() >= 3); + assert!(calibration.variants.iter().all(|v| v.samples_ms.len() == 3)); + assert_eq!(executor.solve(Arc::clone(&input)), solve_cpu_serial(&input)); + assert_eq!( + executor.jobs_completed(), + calibration.variants.len() * 3 + 2 + ); + assert_eq!( + executor.worker_limit(&input, calibration.selected_mode), + calibration.selected_workers + ); + assert!(executor + .set_worker_limit(&input, CpuMode::Functions, 0) + .is_err()); + assert!(executor + .set_worker_limit(&input, CpuMode::Functions, executor.worker_count() + 1) + .is_err()); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/frontend.rs b/experiments/gpu-rust-compiler/compiler/src/frontend.rs new file mode 100644 index 00000000000..24c03f562e0 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/frontend.rs @@ -0,0 +1,941 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A strict frontend for the lab's explicit Rust subset. +//! +//! Scalar i64/bool functions, lexical locals, calls, conditionals and loops are +//! supported. This is not a parser for full Rust. Unsupported syntax is rejected +//! with a source location, before the backend receives a typed program. + +use std::collections::{HashMap, HashSet}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Type { + I64, + Bool, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Program { + pub functions: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Function { + pub name: String, + pub params: Vec<(String, Type)>, + pub return_type: Type, + pub body: Vec, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Stmt { + Let { + name: String, + ty: Option, + mutable: bool, + value: Expr, + }, + Assign { + name: String, + value: Expr, + }, + If { + condition: Expr, + then_body: Vec, + else_body: Vec, + }, + While { + condition: Expr, + body: Vec, + }, + Return(Expr), + Expr(Expr), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Expr { + Int(i64), + Bool(bool), + Var(String), + Call { + name: String, + args: Vec, + }, + Unary { + op: String, + value: Box, + }, + Binary { + op: String, + left: Box, + right: Box, + }, +} + +#[derive(Clone, Debug, PartialEq)] +enum Kind { + Ident(String), + Number(u64), + Symbol(String), + End, +} + +#[derive(Clone, Debug)] +struct Token { + kind: Kind, + offset: usize, +} + +fn location(source: &str, offset: usize, message: &str) -> String { + let prefix = &source[..offset.min(source.len())]; + let line = prefix.bytes().filter(|&b| b == b'\n').count() + 1; + let column = prefix.rsplit('\n').next().unwrap_or("").chars().count() + 1; + format!("line {line}, column {column}: {message}") +} + +fn lex(source: &str) -> Result, String> { + let bytes = source.as_bytes(); + let mut tokens = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes[i].is_ascii_whitespace() { + i += 1; + continue; + } + if bytes[i..].starts_with(b"//") { + while i < bytes.len() && bytes[i] != b'\n' { + i += 1; + } + continue; + } + if bytes[i..].starts_with(b"/*") { + let start = i; + i += 2; + let mut depth = 1; + while i < bytes.len() && depth > 0 { + if bytes[i..].starts_with(b"/*") { + depth += 1; + i += 2; + } else if bytes[i..].starts_with(b"*/") { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + if depth != 0 { + return Err(location(source, start, "unterminated block comment")); + } + continue; + } + let start = i; + let kind = if bytes[i].is_ascii_alphabetic() || bytes[i] == b'_' { + i += 1; + while i < bytes.len() && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') { + i += 1; + } + Kind::Ident(source[start..i].to_owned()) + } else if bytes[i].is_ascii_digit() { + i += 1; + while i < bytes.len() && (bytes[i].is_ascii_digit() || bytes[i] == b'_') { + i += 1; + } + if i < bytes.len() && bytes[i].is_ascii_alphabetic() { + return Err(location( + source, + start, + "only decimal i64 literals without suffixes are supported", + )); + } + let number = source[start..i] + .replace('_', "") + .parse::() + .map_err(|_| location(source, start, "integer literal exceeds the i64 range"))?; + if number > i64::MAX as u64 + 1 { + return Err(location( + source, + start, + "integer literal exceeds the i64 range", + )); + } + Kind::Number(number) + } else { + let two = if i + 1 < bytes.len() { + &bytes[i..i + 2] + } else { + &[] + }; + let multi = match two { + b"->" => Some("->"), + b"==" => Some("=="), + b"!=" => Some("!="), + b"<=" => Some("<="), + b">=" => Some(">="), + b"&&" => Some("&&"), + b"||" => Some("||"), + b"<<" => Some("<<"), + b">>" => Some(">>"), + b"+=" => Some("+="), + b"-=" => Some("-="), + b"*=" => Some("*="), + b"/=" => Some("/="), + b"%=" => Some("%="), + b"&=" => Some("&="), + b"|=" => Some("|="), + b"^=" => Some("^="), + b"::" => { + return Err(location( + source, + start, + "paths and associated methods are outside the supported Rust subset", + )) + } + _ => None, + }; + if let Some(symbol) = multi { + i += 2; + Kind::Symbol(symbol.to_owned()) + } else if b"(){}:,;=+-*/%!&|^<>".contains(&bytes[i]) { + i += 1; + Kind::Symbol((bytes[start] as char).to_string()) + } else { + return Err(location(source, start, + "unsupported syntax; references, strings, attributes, macros, arrays and full Rust types are not supported")); + } + }; + tokens.push(Token { + kind, + offset: start, + }); + } + tokens.push(Token { + kind: Kind::End, + offset: source.len(), + }); + Ok(tokens) +} + +struct Parser<'a> { + source: &'a str, + tokens: Vec, + cursor: usize, +} + +impl Parser<'_> { + fn token(&self) -> &Kind { + &self.tokens[self.cursor].kind + } + fn error(&self, message: &str) -> String { + location(self.source, self.tokens[self.cursor].offset, message) + } + fn is(&self, text: &str) -> bool { + matches!(self.token(), Kind::Ident(value) | Kind::Symbol(value) if value == text) + } + fn take(&mut self, text: &str) -> bool { + if self.is(text) { + self.cursor += 1; + true + } else { + false + } + } + fn expect(&mut self, text: &str) -> Result<(), String> { + if self.take(text) { + Ok(()) + } else { + Err(self.error(&format!("expected '{text}'"))) + } + } + fn identifier(&mut self) -> Result { + match self.token().clone() { + Kind::Ident(name) if !is_keyword(&name) && name != "_" => { + self.cursor += 1; + Ok(name) + } + _ => Err(self.error("expected an identifier")), + } + } + fn ty(&mut self) -> Result { + if self.take("i64") { + Ok(Type::I64) + } else if self.take("bool") { + Ok(Type::Bool) + } else { + Err(self.error("only i64 and bool types are supported")) + } + } + fn program(&mut self) -> Result { + let mut functions = Vec::new(); + while self.token() != &Kind::End { + if !self.take("fn") { + return Err(self + .error("expected a function: fn name(...); other Rust items are unsupported")); + } + let name = self.identifier()?; + self.expect("(")?; + let mut params = Vec::new(); + if !self.take(")") { + loop { + let parameter = self.identifier()?; + self.expect(":")?; + params.push((parameter, self.ty()?)); + if self.take(")") { + break; + } + self.expect(",")?; + if self.take(")") { + break; + } + } + } + let explicit_return_type = self.take("->"); + let return_type = if explicit_return_type { + self.ty()? + } else if name == "main" { + Type::I64 + } else { + return Err( + self.error("non-main functions need an explicit -> i64 or -> bool return type") + ); + }; + let mut body = self.block(true)?; + if name == "main" && !explicit_return_type && !always_returns(&body) { + body.push(Stmt::Return(Expr::Int(0))); + } + functions.push(Function { + name, + params, + return_type, + body, + }); + } + if functions.is_empty() { + return Err(self.error("source contains no functions")); + } + Ok(Program { functions }) + } + fn block(&mut self, allow_tail: bool) -> Result, String> { + self.expect("{")?; + let mut body = Vec::new(); + while !self.take("}") { + if self.token() == &Kind::End { + return Err(self.error("unterminated block")); + } + if self.take("let") { + let mutable = self.take("mut"); + let name = self.identifier()?; + let ty = if self.take(":") { + Some(self.ty()?) + } else { + None + }; + self.expect("=")?; + let value = self.expr(0)?; + self.expect(";")?; + body.push(Stmt::Let { + name, + ty, + mutable, + value, + }); + } else if self.take("while") { + let condition = self.expr(0)?; + let loop_body = self.block(false)?; + body.push(Stmt::While { + condition, + body: loop_body, + }); + } else if self.take("if") { + body.push(self.if_statement()?); + } else if self.take("return") { + let value = self.expr(0)?; + self.expect(";")?; + body.push(Stmt::Return(value)); + } else { + // Assignment is a statement in this subset, never an expression. + let assignment = matches!(self.token(), Kind::Ident(_)) + && matches!(&self.tokens[self.cursor + 1].kind, Kind::Symbol(op) + if matches!(op.as_str(), "=" | "+=" | "-=" | "*=" | "/=" | "%=" | "&=" | "|=" | "^=")); + if assignment { + let name = self.identifier()?; + let op = match self.token().clone() { + Kind::Symbol(op) => op, + _ => unreachable!(), + }; + self.cursor += 1; + let mut value = self.expr(0)?; + if op != "=" { + value = Expr::Binary { + op: op[..1].to_owned(), + left: Box::new(Expr::Var(name.clone())), + right: Box::new(value), + }; + } + self.expect(";")?; + body.push(Stmt::Assign { name, value }); + } else { + let value = self.expr(0)?; + if self.take(";") { + body.push(Stmt::Expr(value)); + } else if allow_tail && self.take("}") { + body.push(Stmt::Return(value)); + return Ok(body); + } else { + return Err(self.error("expected ';'; implicit tail values are supported only in function bodies")); + } + } + } + } + Ok(body) + } + fn if_statement(&mut self) -> Result { + let condition = self.expr(0)?; + let then_body = self.block(false)?; + let else_body = if self.take("else") { + if self.take("if") { + vec![self.if_statement()?] + } else { + self.block(false)? + } + } else { + Vec::new() + }; + Ok(Stmt::If { + condition, + then_body, + else_body, + }) + } + fn expr(&mut self, minimum: u8) -> Result { + let mut left = if self.take("-") { + if let Kind::Number(value) = self.token() { + if *value == i64::MAX as u64 + 1 { + self.cursor += 1; + Expr::Int(i64::MIN) + } else { + Expr::Unary { + op: "-".to_owned(), + value: Box::new(self.expr(10)?), + } + } + } else { + Expr::Unary { + op: "-".to_owned(), + value: Box::new(self.expr(10)?), + } + } + } else if self.take("!") { + Expr::Unary { + op: "!".to_owned(), + value: Box::new(self.expr(10)?), + } + } else if self.take("(") { + let inner = self.expr(0)?; + self.expect(")")?; + inner + } else { + match self.token().clone() { + Kind::Number(value) if value <= i64::MAX as u64 => { + self.cursor += 1; + Expr::Int(value as i64) + } + Kind::Number(_) => return Err(self.error("positive integer literal exceeds i64::MAX")), + Kind::Ident(value) if value == "true" || value == "false" => { + self.cursor += 1; + Expr::Bool(value == "true") + } + Kind::Ident(_) => { + let name = self.identifier()?; + if self.is("!") { return Err(self.error("macros are outside the supported Rust subset")); } + if self.take("(") { + let mut args = Vec::new(); + if !self.take(")") { + loop { + args.push(self.expr(0)?); + if self.take(")") { break; } + self.expect(",")?; + if self.take(")") { break; } + } + } + Expr::Call { name, args } + } else { Expr::Var(name) } + } + _ => return Err(self.error("expected a scalar expression; references and other Rust expressions are unsupported")), + } + }; + let mut compared = false; + while let Kind::Symbol(op) = self.token() { + let op = op.clone(); + let precedence = match op.as_str() { + "||" => 1, + "&&" => 2, + "==" | "!=" | "<" | "<=" | ">" | ">=" => 3, + "|" => 4, + "^" => 5, + "&" => 6, + "<<" | ">>" => 7, + "+" | "-" => 8, + "*" | "/" | "%" => 9, + _ => break, + }; + if precedence < minimum { + break; + } + if precedence == 3 && compared { + return Err( + self.error("comparison operators cannot be chained without parentheses") + ); + } + if precedence == 3 { + compared = true; + } + self.cursor += 1; + let right = self.expr(precedence + 1)?; + left = Expr::Binary { + op, + left: Box::new(left), + right: Box::new(right), + }; + } + Ok(left) + } +} + +fn is_keyword(name: &str) -> bool { + matches!( + name, + "fn" | "let" + | "mut" + | "while" + | "if" + | "else" + | "return" + | "true" + | "false" + | "break" + | "continue" + | "for" + | "loop" + | "match" + | "struct" + | "enum" + | "impl" + | "trait" + | "use" + | "pub" + | "const" + | "static" + | "unsafe" + | "async" + | "await" + | "move" + | "ref" + | "self" + | "Self" + | "as" + | "crate" + | "dyn" + | "extern" + | "in" + | "mod" + | "super" + | "type" + | "where" + | "abstract" + | "become" + | "box" + | "do" + | "final" + | "gen" + | "macro" + | "override" + | "priv" + | "try" + | "typeof" + | "unsized" + | "virtual" + | "yield" + ) +} + +fn always_returns(body: &[Stmt]) -> bool { + body.iter().any(|statement| match statement { + Stmt::Return(_) => true, + Stmt::If { + then_body, + else_body, + .. + } => !else_body.is_empty() && always_returns(then_body) && always_returns(else_body), + _ => false, + }) +} + +#[derive(Clone, Copy)] +struct Binding { + ty: Type, + mutable: bool, +} +type Signatures = HashMap, Type)>; + +struct Checker<'a> { + signatures: &'a Signatures, + scopes: Vec>, + declared: HashSet, + return_type: Type, + function: &'a str, +} + +impl Checker<'_> { + fn error(&self, message: impl AsRef) -> String { + format!("function '{}': {}", self.function, message.as_ref()) + } + fn lookup(&self, name: &str) -> Result { + self.scopes + .iter() + .rev() + .find_map(|scope| scope.get(name).copied()) + .ok_or_else(|| self.error(format!("unknown or out-of-scope local '{name}'"))) + } + fn expression(&self, expr: &Expr) -> Result { + match expr { + Expr::Int(_) => Ok(Type::I64), + Expr::Bool(_) => Ok(Type::Bool), + Expr::Var(name) => Ok(self.lookup(name)?.ty), + Expr::Call { name, args } => { + if self.scopes.iter().any(|scope| scope.contains_key(name)) { + return Err(self.error(format!( + "scalar local '{name}' shadows the function and cannot be called" + ))); + } + if name == "print_i64" { + return Err( + self.error("print_i64 has no value and is only supported as a statement") + ); + } + let (parameters, result) = self + .signatures + .get(name) + .ok_or_else(|| self.error(format!("unknown function '{name}'")))?; + if args.len() != parameters.len() { + return Err(self.error(format!( + "function '{name}' expects {} arguments, got {}", + parameters.len(), + args.len() + ))); + } + for (argument, expected) in args.iter().zip(parameters) { + let actual = self.expression(argument)?; + if actual != *expected { + return Err(self.error(format!( + "argument to '{name}' has type {actual:?}, expected {expected:?}" + ))); + } + } + Ok(*result) + } + Expr::Unary { op, value } => { + let ty = self.expression(value)?; + match (op.as_str(), ty) { + ("-", Type::I64) => Ok(Type::I64), + ("!", Type::Bool) => Ok(Type::Bool), + ("!", Type::I64) => Ok(Type::I64), + _ => Err(self.error(format!("unary '{op}' is invalid for {ty:?}"))), + } + } + Expr::Binary { op, left, right } => { + let lhs = self.expression(left)?; + let rhs = self.expression(right)?; + match op.as_str() { + "&&" | "||" if lhs == Type::Bool && rhs == Type::Bool => Ok(Type::Bool), + "==" | "!=" if lhs == rhs => Ok(Type::Bool), + "<" | "<=" | ">" | ">=" if lhs == Type::I64 && rhs == Type::I64 => { + Ok(Type::Bool) + } + "&" | "|" | "^" if lhs == rhs => Ok(lhs), + "+" | "-" | "*" | "/" | "%" | "<<" | ">>" + if lhs == Type::I64 && rhs == Type::I64 => + { + Ok(Type::I64) + } + _ => Err(self.error(format!( + "operator '{op}' is invalid for {lhs:?} and {rhs:?}" + ))), + } + } + } + } + fn body(&mut self, body: &[Stmt]) -> Result<(), String> { + self.scopes.push(HashMap::new()); + for statement in body { + match statement { + Stmt::Let { + name, + ty, + mutable, + value, + } => { + let actual = self.expression(value)?; + if let Some(expected) = ty { + if actual != *expected { + return Err(self.error(format!( + "local '{name}' has type {actual:?}, annotated {expected:?}" + ))); + } + } + // Reusing spelling needs binder IDs in the backend; reject + // shadowing until that representation is implemented. + if !self.declared.insert(name.clone()) { + return Err(self.error(format!( + "local shadowing/redeclaration of '{name}' is not supported" + ))); + } + self.scopes.last_mut().unwrap().insert( + name.clone(), + Binding { + ty: actual, + mutable: *mutable, + }, + ); + } + Stmt::Assign { name, value } => { + let binding = self.lookup(name)?; + if !binding.mutable { + return Err(self.error(format!( + "cannot assign to immutable local '{name}'; use let mut" + ))); + } + if self.expression(value)? != binding.ty { + return Err(self.error(format!("assignment changes the type of '{name}'"))); + } + } + Stmt::If { + condition, + then_body, + else_body, + } => { + if self.expression(condition)? != Type::Bool { + return Err(self.error("if condition must be bool")); + } + self.body(then_body)?; + self.body(else_body)?; + } + Stmt::While { condition, body } => { + if self.expression(condition)? != Type::Bool { + return Err(self.error("while condition must be bool")); + } + self.body(body)?; + } + Stmt::Return(value) => { + if self.expression(value)? != self.return_type { + return Err(self.error("return value has the wrong type")); + } + } + Stmt::Expr(Expr::Call { name, args }) if name == "print_i64" => { + if self.scopes.iter().any(|scope| scope.contains_key(name)) { + return Err(self.error("scalar local 'print_i64' cannot be called")); + } + if args.len() != 1 || self.expression(&args[0])? != Type::I64 { + return Err(self.error("print_i64 expects one i64 argument")); + } + } + Stmt::Expr(expr) => { + self.expression(expr)?; + } + } + } + self.scopes.pop(); + Ok(()) + } +} + +fn validate(program: &Program) -> Result<(), String> { + let mut signatures = Signatures::new(); + for function in &program.functions { + if function.name == "print_i64" { + return Err("print_i64 is a reserved host builtin".to_owned()); + } + if signatures + .insert( + function.name.clone(), + ( + function.params.iter().map(|(_, ty)| *ty).collect(), + function.return_type, + ), + ) + .is_some() + { + return Err(format!("duplicate function '{}'", function.name)); + } + if function.name == "main" && !function.params.is_empty() { + return Err("the experimental main entry must have no parameters".to_owned()); + } + } + for function in &program.functions { + let mut parameters = HashMap::new(); + for (name, ty) in &function.params { + if parameters + .insert( + name.clone(), + Binding { + ty: *ty, + mutable: false, + }, + ) + .is_some() + { + return Err(format!( + "function '{}': duplicate parameter '{name}'", + function.name + )); + } + } + let mut checker = Checker { + signatures: &signatures, + declared: parameters.keys().cloned().collect(), + scopes: vec![parameters], + return_type: function.return_type, + function: &function.name, + }; + checker.body(&function.body)?; + if !always_returns(&function.body) { + return Err(format!( + "function '{}': every path must return {:?}", + function.name, function.return_type + )); + } + } + Ok(()) +} + +pub fn parse(source: &str) -> Result { + let tokens = lex(source)?; + let program = Parser { + source, + tokens, + cursor: 0, + } + .program()?; + validate(&program)?; + Ok(program) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn typed_functions_loops_calls_compound_assignments_and_tail_values() { + let source = r#" + fn sum(n: i64) -> i64 { + let mut i = 0; + let mut result: i64 = 0; + while i < n { result += i; i += 1; } + result + } + fn positive(n: i64) -> bool { return n > 0; } + fn main() -> i64 { + let answer = sum(10); + if positive(answer) && answer == 45 { print_i64(answer); } + else { print_i64(-1); } + return 0; + } + "#; + let program = parse(source).unwrap(); + assert_eq!(program.functions.len(), 3); + assert!( + matches!(program.functions[0].body.last(), Some(Stmt::Return(Expr::Var(name))) if name == "result") + ); + assert!(matches!(program.functions[0].body[2], Stmt::While { .. })); + } + + #[test] + fn pratt_precedence_and_short_circuit_ast_are_preserved() { + let program = parse("fn f() -> bool { 1 + 2 * 3 == 7 || false && true }").unwrap(); + let Stmt::Return(Expr::Binary { op, left, right }) = &program.functions[0].body[0] else { + panic!() + }; + assert_eq!(op, "||"); + assert!(matches!(&**right, Expr::Binary { op, .. } if op == "&&")); + let Expr::Binary { left: addition, .. } = &**left else { + panic!() + }; + assert!(matches!(&**addition, Expr::Binary { op, right, .. } + if op == "+" && matches!(&**right, Expr::Binary { op, .. } if op == "*"))); + } + + #[test] + fn integer_bounds_and_nested_comments() { + let program = + parse("/* outer /* nested */ */ fn f() -> i64 { -9223372036854775808 }").unwrap(); + assert!(matches!( + program.functions[0].body[0], + Stmt::Return(Expr::Int(i64::MIN)) + )); + assert!(parse("fn f() -> i64 { 9223372036854775808 }").is_err()); + assert!(parse("fn f() -> i64 { 18446744073709551616 }").is_err()); + assert!(parse("/* open").unwrap_err().contains("unterminated")); + } + + #[test] + fn main_without_return_annotation_uses_host_wrapper_zero() { + let program = parse("fn main() { print_i64(42); }").unwrap(); + assert!(matches!( + program.functions[0].body.last(), + Some(Stmt::Return(Expr::Int(0))) + )); + } + + #[test] + fn validates_mutability_types_scope_calls_and_return_paths() { + for source in [ + "fn main() { let x = 1; x = 2; }", + "fn main() { let mut x = 1; x = true; }", + "fn main() { let x: bool = 1; }", + "fn main() { while 1 { print_i64(0); } }", + "fn main() { if true { let x = 1; } print_i64(x); }", + "fn main() { print_i64(true); }", + "fn main() { let x = print_i64(1); }", + "fn main() { missing(1); }", + "fn f(x: bool) -> i64 { 1 } fn main() { f(2); }", + "fn f(x: i64) -> i64 { x } fn main() { f(); }", + "fn f() -> i64 { 1 } fn main() { let f = 1; f(); }", + "fn main() { let print_i64 = 1; print_i64(2); }", + "fn f(x: bool) -> i64 { if x { return 1; } }", + "fn main() { let x = 1; let x = 2; }", + "fn f(x: i64, x: i64) -> i64 { x }", + "fn f() -> i64 { 1 } fn f() -> i64 { 2 }", + "fn main() -> i64 { print_i64(42); }", + ] { + assert!( + parse(source).is_err(), + "accepted invalid subset source: {source}" + ); + } + parse("fn f(x: bool) -> i64 { if x { return 1; } else { return 2; } }").unwrap(); + parse("fn main() -> bool { true }").unwrap(); + } + + #[test] + fn rejects_features_instead_of_claiming_full_rust() { + for source in [ + "fn main() { println!(1); }", + "fn f(x: &i64) -> i64 { 0 }", + "fn f() -> i32 { 1 }", + "fn main() { let x = &1; }", + "fn main() { let x = [1, 2]; }", + "fn main() { let x = 1u64; }", + "fn main() { let x = 0xff; }", + "fn main() { let x = 1.wrapping_add(2); }", + "fn main() { let x = 1 < 2 < 3; }", + "fn main() { break; }", + "struct Thing { x: i64 }", + "fn type() -> i64 { 1 }", + ] { + assert!( + parse(source).is_err(), + "accepted unsupported Rust: {source}" + ); + } + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/glc.rs b/experiments/gpu-rust-compiler/compiler/src/glc.rs new file mode 100644 index 00000000000..00394ca791d --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/glc.rs @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validated GLC1 workpacks shared with the experimental CUDA executable. + +use crate::analysis::{FunctionDesc, GPUGroup, PackedAnalysis}; +use std::collections::BTreeMap; +use std::time::Instant; + +#[cfg(test)] +pub fn decode(bytes: &[u8]) -> Result { + decode_timed(bytes).map(|(packed, _, _)| packed) +} + +pub fn decode_timed(bytes: &[u8]) -> Result<(PackedAnalysis, f64, f64), String> { + let decode_start = Instant::now(); + if bytes.get(..4) != Some(b"GLC1") { + return Err("Expected GLC1 input magic".into()); + } + let mut reader = Reader { bytes, position: 4 }; + let functions = reader.word()? as usize; + let blocks = reader.word()? as usize; + let cells = reader.word()? as usize; + let groups = reader.word()? as usize; + let max_blocks = reader.word()? as usize; + let edges = reader.word()? as usize; + if functions == 0 || blocks == 0 || cells == 0 || groups == 0 { + return Err("Empty workload dimensions".into()); + } + let expected_words = groups as u64 * 5 + blocks as u64 + 1 + edges as u64 + cells as u64 * 3; + if expected_words * 4 != (bytes.len() - reader.position) as u64 { + return Err("Binary length does not match header dimensions".into()); + } + let descriptors = (0..groups) + .map(|_| { + Ok(GPUGroup { + block_base: reader.word()?, + block_count: reader.word()?, + row_base: reader.word()?, + words: reader.word()?, + word: reader.word()?, + }) + }) + .collect::, String>>()?; + let successor_offsets = reader.words(blocks + 1)?; + let successors = reader.words(edges)?; + let uses = reader.words(cells)?; + let defs = reader.words(cells)?; + let phi_out = reader.words(cells)?; + if successor_offsets[0] != 0 || successor_offsets[blocks] as usize != edges { + return Err("Successor offsets do not span the edges array".into()); + } + if successor_offsets + .windows(2) + .any(|pair| pair[0] > pair[1] || pair[1] as usize > edges) + { + return Err("Invalid successor offsets".into()); + } + + struct Shape { + blocks: usize, + row: usize, + words: usize, + seen: Vec, + } + let mut shapes = BTreeMap::new(); + let mut largest = 0; + for group in &descriptors { + let block_base = group.block_base as usize; + let block_count = group.block_count as usize; + let row = group.row_base as usize; + let words = group.words as usize; + let word = group.word as usize; + if block_count == 0 + || words == 0 + || words > groups + || word >= words + || block_base as u64 + block_count as u64 > blocks as u64 + || row as u64 + block_count as u64 * words as u64 > cells as u64 + { + return Err("Invalid function-word group".into()); + } + largest = largest.max(block_count); + let shape = shapes.entry(block_base).or_insert_with(|| Shape { + blocks: block_count, + row, + words, + seen: vec![false; words], + }); + if shape.blocks != block_count + || shape.row != row + || shape.words != words + || shape.seen[word] + { + return Err("Duplicate or inconsistent function-word group".into()); + } + shape.seen[word] = true; + } + if shapes.len() != functions || largest != max_blocks { + return Err("Function count or maximum block count does not match descriptors".into()); + } + let mut function_descs = Vec::with_capacity(functions); + let mut row_offsets = Vec::with_capacity(blocks + 1); + let (mut next_block, mut next_cell) = (0, 0); + for (base, shape) in shapes { + if base != next_block || shape.row != next_cell || shape.seen.contains(&false) { + return Err( + "Function groups do not cover every block and output cell exactly once".into(), + ); + } + let end = base + shape.blocks; + for block in base..end { + for &successor in &successors + [successor_offsets[block] as usize..successor_offsets[block + 1] as usize] + { + let successor = successor as usize; + if successor < base || successor >= end { + return Err("CFG successor escapes its owning function".into()); + } + } + row_offsets.push(shape.row + (block - base) * shape.words); + } + function_descs.push(FunctionDesc { + module_index: function_descs.len(), + block_base: base, + block_count: shape.blocks, + row_base: shape.row, + words: shape.words, + value_count: shape.words * 32, + }); + next_block = end; + next_cell += shape.blocks * shape.words; + } + if next_block != blocks || next_cell != cells { + return Err("Function groups leave uncovered blocks or cells".into()); + } + row_offsets.push(cells); + let input_decode_ms = decode_start.elapsed().as_secs_f64() * 1_000.0; + let predecessor_start = Instant::now(); + + // Build predecessor CSR once. Solvers benchmark resident input rather + // than paying file decoding or reverse-edge construction on each pass. + let mut predecessor_offsets = vec![0u32; blocks + 1]; + for &successor in &successors { + predecessor_offsets[successor as usize + 1] += 1; + } + for block in 0..blocks { + predecessor_offsets[block + 1] += predecessor_offsets[block]; + } + let mut cursor = predecessor_offsets[..blocks].to_vec(); + let mut predecessors = vec![0; edges]; + for block in 0..blocks { + for &successor in + &successors[successor_offsets[block] as usize..successor_offsets[block + 1] as usize] + { + let successor = successor as usize; + predecessors[cursor[successor] as usize] = block as u32; + cursor[successor] += 1; + } + } + let predecessor_prepare_ms = predecessor_start.elapsed().as_secs_f64() * 1_000.0; + Ok(( + PackedAnalysis { + functions: function_descs, + groups: descriptors, + successor_offsets, + successors, + predecessor_offsets, + predecessors, + row_offsets, + uses, + defs, + phi_out, + total_cells: cells, + }, + input_decode_ms, + predecessor_prepare_ms, + )) +} + +struct Reader<'a> { + bytes: &'a [u8], + position: usize, +} + +impl Reader<'_> { + fn word(&mut self) -> Result { + let raw = self + .bytes + .get(self.position..self.position + 4) + .ok_or("Truncated input")?; + self.position += 4; + Ok(u32::from_le_bytes(raw.try_into().unwrap())) + } + + fn words(&mut self, count: usize) -> Result, String> { + (0..count).map(|_| self.word()).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::{solve_cpu_parallel, solve_cpu_serial, solve_cpu_word_parallel}; + + fn encode(words: &[u32]) -> Vec { + let mut bytes = b"GLC1".to_vec(); + bytes.extend(words.iter().flat_map(|word| word.to_le_bytes())); + bytes + } + + #[test] + fn native_solvers_keep_phi_uses_and_definitions_correct_across_a_loop() { + // Two blocks and two bitset words. Block zero defines bit 1; + // block one uses it. A phi operand on block zero uses bit 2. + let bytes = encode(&[ + 1, 2, 4, 2, 2, 2, // counts + 0, 2, 0, 2, 0, 0, 2, 0, 2, 1, // groups + 0, 1, 2, // successor offsets + 1, 0, // loop successors + 1, 0, 2, 8, // uses + 2, 0, 0, 0, // definitions + 4, 16, 0, 0, // phi-out uses + ]); + let packed = decode(&bytes).unwrap(); + // OUT0=(IN1|PHI0), then DEF0 kills bit 1. The loop carries + // both words to block one, where its local use restores bit 1. + let expected = vec![5, 24, 7, 24]; + assert_eq!(solve_cpu_serial(&packed), expected); + assert_eq!(solve_cpu_parallel(&packed), expected); + assert_eq!(solve_cpu_word_parallel(&packed), expected); + } + + #[test] + fn workpack_decoder_rejects_truncation_and_invalid_graph_ownership() { + let words = vec![ + 2, 2, 2, 2, 1, 0, // two isolated one-block functions + 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, // offsets, then no edges + 1, 2, 0, 0, 0, 0, + ]; + let valid = encode(&words); + assert_eq!(solve_cpu_serial(&decode(&valid).unwrap()), vec![1, 2]); + for length in 0..valid.len() { + assert!( + decode(&valid[..length]).is_err(), + "Accepted truncated length {length}" + ); + } + let mut trailing = valid.clone(); + trailing.push(0); + assert!(decode(&trailing).is_err()); + let mut wrong_magic = valid.clone(); + wrong_magic[0] = b'X'; + assert!(decode(&wrong_magic).is_err()); + for (index, value) in [(0, 0), (4, 2), (6, 1), (11, 0), (13, 0)] { + let mut malformed = words.clone(); + malformed[index] = value; + assert!( + decode(&encode(&malformed)).is_err(), + "Accepted malformed word {index}" + ); + } + let crossing = encode(&[ + 2, 2, 2, 2, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, + 1, // function zero has an edge to function one + 1, 2, 0, 0, 0, 0, + ]); + assert!(decode(&crossing) + .unwrap_err() + .contains("escapes its owning function")); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/ir.rs b/experiments/gpu-rust-compiler/compiler/src/ir.rs new file mode 100644 index 00000000000..e4dceb858ec --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/ir.rs @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Compiler-owned control flow and operand identities, recorded during emission. +//! Text is retained only for final LLVM output; analysis never reparses it. + +#[derive(Clone, Debug)] +pub struct ModuleIR { + pub text: String, + pub preamble: String, + pub functions: Vec, + pub postamble: String, +} + +#[derive(Clone, Debug)] +pub struct FunctionIR { + pub name: String, + pub header: String, + pub footer: String, + pub value_count: usize, + pub blocks: Vec, + pub phi_edges: Vec, +} + +#[derive(Clone, Debug)] +pub struct BlockIR { + pub name: String, + pub instructions: Vec, + pub successors: Vec, +} + +#[derive(Clone, Debug)] +pub struct InstructionIR { + /// Complete indented LLVM instruction, without its trailing newline. + pub text: String, + pub definition: Option, + /// SSA operand IDs. Constants and globals do not require liveness bits. + pub uses: Vec, + /// Only computations which can safely be removed when their result is dead. + pub pure: bool, + pub is_phi: bool, +} + +#[derive(Clone, Debug)] +pub struct PhiEdge { + pub from: usize, + pub to: usize, + pub values: Vec, +} + +impl ModuleIR { + pub fn render(&self) -> String { + let mut text = self.preamble.clone(); + for function in &self.functions { + text.push_str(&function.render()); + } + text.push_str(&self.postamble); + text + } +} + +impl FunctionIR { + pub fn render(&self) -> String { + let mut text = self.header.clone(); + for block in &self.blocks { + text.push_str(&block.name); + text.push_str(":\n"); + for instruction in &block.instructions { + text.push_str(&instruction.text); + text.push('\n'); + } + } + text.push_str(&self.footer); + text + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/leaf_packet.rs b/experiments/gpu-rust-compiler/compiler/src/leaf_packet.rs new file mode 100644 index 00000000000..69862398a69 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/leaf_packet.rs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Strict whole-module GEM1 decoder shared by single-module emission and corpus replay. +//! Rejected LLVM semantics are never approximated or lowered through this schema. +use crate::{ + codegen_service::*, + machine_codegen::{self, MachineTarget}, +}; +use sha2::{Digest, Sha256}; +use std::sync::Arc; + +pub struct DecodedPacket { + pub batch: Arc, + pub target: MachineTarget, + pub llvm_target: String, +} +fn hash(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +struct Reader<'a> { + bytes: &'a [u8], + position: usize, +} +impl Reader<'_> { + fn bytes(&mut self, count: usize) -> Result<&[u8], String> { + let end = self + .position + .checked_add(count) + .ok_or("packet size overflow")?; + let value = self + .bytes + .get(self.position..end) + .ok_or("truncated GEM1 packet")?; + self.position = end; + Ok(value) + } + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.bytes(4)?.try_into().unwrap())) + } + fn i64(&mut self) -> Result { + Ok(i64::from_le_bytes(self.bytes(8)?.try_into().unwrap())) + } + fn string(&mut self) -> Result { + let size = self.u32()? as usize; + if size == 0 || size > 4096 { + return Err("invalid packet string length".into()); + } + String::from_utf8(self.bytes(size)?.to_vec()) + .map_err(|_| "packet string is not UTF-8".into()) + } +} +pub fn decode(bytes: &[u8]) -> Result { + decode_with_key( + bytes, + BatchKey { + logical_id: "llvm-module".into(), + generation: 1, + }, + ) +} + +/// Fresh generations let a retained service replay the same captured module +/// without changing its instruction, ABI or source identity. +pub fn decode_with_key(bytes: &[u8], key: BatchKey) -> Result { + if bytes.len() > 128 * 1024 * 1024 { + return Err("GEM1 input exceeds 128 MiB".into()); + } + let mut reader = Reader { bytes, position: 0 }; + if reader.bytes(4)? != b"GEM1" { + return Err("unsupported leaf packet schema".into()); + } + let triple = reader.string()?; + let target = if triple.starts_with("x86_64-") { + MachineTarget::X86_64 + } else if triple.starts_with("aarch64-") || triple.starts_with("arm64-") { + MachineTarget::Aarch64 + } else { + return Err("unsupported machine architecture".into()); + }; + // Rust target names and LLVM's Darwin triples name the same scalar ABI. + // Preserve the original LLVM spelling in the receipt; only the ABI key is + // normalized. No module target or target instructions are rewritten. + let abi_triple = + if triple.starts_with("arm64-apple-macosx") || triple.starts_with("aarch64-apple-macosx") { + "aarch64-apple-darwin".to_string() + } else if triple.starts_with("x86_64-apple-macosx") { + "x86_64-apple-darwin".to_string() + } else { + triple.clone() + }; + let count = reader.u32()? as usize; + if count == 0 || count > 65536 { + return Err("packet function count is out of bounds".into()); + } + let mut functions = Vec::new(); + let mut instructions = Vec::new(); + let mut symbols = Vec::new(); + let mut abis = Vec::new(); + for index in 0..count { + let name = reader.string()?; + let params = reader.u32()?; + let unwind = reader.u32()?; + let values = reader.u32()?; + let ops = reader.u32()?; + if params > 1 || unwind > 1 || values == 0 || values > 256 || ops == 0 || ops > 4096 { + return Err("leaf function dimensions are out of bounds".into()); + } + let start = u32::try_from(instructions.len()).map_err(|_| "instruction count overflow")?; + for _ in 0..ops { + let opcode = match reader.u32()? { + 0 => Opcode::Argument, + 1 => Opcode::Const, + 2 => Opcode::Copy, + 3 => Opcode::Add, + 4 => Opcode::Sub, + 5 => Opcode::Mul, + 6 => Opcode::Return, + _ => return Err("unsupported leaf opcode".into()), + }; + let result = reader.u32()?; + let a = reader.u32()?; + let b = reader.u32()?; + let immediate = reader.i64()?; + instructions.push(Instruction { + opcode, + result: (result != u32::MAX).then_some(result), + operands: [a, b], + immediate, + }); + } + functions.push(Function { + symbol: index as u32, + instruction_start: start, + instruction_count: ops, + value_count: values, + abi: index as u32, + }); + symbols.push(Symbol { name }); + abis.push(Abi { + calling_convention: if target == MachineTarget::Aarch64 && abi_triple.contains("apple") + { + CallingConvention::AppleAarch64 + } else { + CallingConvention::SystemV + }, + parameter_count: params, + returns_i64: true, + requires_unwind: unwind == 1, + }); + } + if reader.position != bytes.len() { + return Err("trailing bytes in GEM1 packet".into()); + } + let batch = Arc::new(FlatBatch::new( + BatchIdentity { + workload_id: hash(bytes), + toolchain_id: "rust1.98.1-llvm22.1.8-GEM1".into(), + policy_id: "fast-leaf-fixedframe-v1".into(), + target_abi: abi_triple, + source_revision: hash(include_bytes!("machine_codegen.rs")), + }, + key, + functions, + instructions, + symbols, + abis, + )?); + machine_codegen::pack(&batch, target)?; + Ok(DecodedPacket { + batch, + target, + llvm_target: triple, + }) +} diff --git a/experiments/gpu-rust-compiler/compiler/src/lib.rs b/experiments/gpu-rust-compiler/compiler/src/lib.rs new file mode 100644 index 00000000000..e3e4c051bfb --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/lib.rs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub mod bitcode_linker; +pub mod cargo_policy; +pub mod codegen_service; +pub mod leaf_packet; +pub mod machine_codegen; +pub mod macho_unwind; +pub mod native_codegen; diff --git a/experiments/gpu-rust-compiler/compiler/src/llvm.rs b/experiments/gpu-rust-compiler/compiler/src/llvm.rs new file mode 100644 index 00000000000..17fd2a0001b --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/llvm.rs @@ -0,0 +1,879 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! LLVM emission for the experimental compiler's small Rust language subset. +//! Integer arithmetic wraps. Shifts mask their count to six bits. Division and +//! remainder trap on zero and MIN / -1 rather than invoking LLVM undefined behavior. +use crate::frontend::{Expr, Function, Program, Stmt, Type}; +use crate::ir::{BlockIR, FunctionIR, InstructionIR, ModuleIR, PhiEdge}; +use std::collections::{HashMap, HashSet}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Ty { + I64, + Bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::frontend; + + #[test] + fn structured_render_preserves_legacy_llvm() { + let program = frontend::parse("fn main() -> i64 { return 17; }").unwrap(); + let module = emit_structured(&program).unwrap(); + let expected = concat!( + "; Experimental Rust subset compiler: target-neutral LLVM IR\n", + "@.format = private unnamed_addr constant [6 x i8] c\"%lld\\0A\\00\", align 1\n", + "declare i32 @printf(ptr, ...)\n", + "declare void @llvm.trap() cold noreturn nounwind\n\n", + "define i64 @r_main() {\nentry:\n ret i64 17\n}\n\n", + "define i32 @main() {\nentry:\n %result = call i64 @r_main()\n", + " %printed = call i32 (ptr, ...) @printf(ptr @.format, i64 %result)\n", + " ret i32 0\n}\n", + ); + assert_eq!(module.text, expected); + assert_eq!(module.render(), expected); + } + + #[test] + fn nested_phi_operands_are_attached_to_predecessor_edges() { + let program = frontend::parse( + "fn choose(a: i64, b: i64) -> bool { return a > 0 && (b > 1 || a < 9); } + fn main() -> bool { return choose(3, 4); }", + ) + .unwrap(); + let module = emit_structured(&program).unwrap(); + let function = &module.functions[0]; + let phi_count = function + .blocks + .iter() + .flat_map(|block| &block.instructions) + .filter(|instruction| instruction.is_phi) + .count(); + assert_eq!(phi_count, 2); + assert_eq!(function.phi_edges.len(), 2); + for edge in &function.phi_edges { + assert!(function.blocks[edge.from].successors.contains(&edge.to)); + assert_eq!(edge.values.len(), 1); + assert!(function.blocks[edge.from] + .instructions + .iter() + .any(|instruction| instruction.definition == Some(edge.values[0]))); + assert!(function.blocks[edge.to] + .instructions + .iter() + .any(|instruction| instruction.is_phi && instruction.uses.is_empty())); + } + let mut definitions = HashSet::new(); + for instruction in function.blocks.iter().flat_map(|block| &block.instructions) { + if let Some(id) = instruction.definition { + assert!(id >= 2 && id < function.value_count); + assert!(definitions.insert(id)); + } + assert!(instruction.uses.iter().all(|id| *id < function.value_count)); + } + assert_eq!(definitions.len() + 2, function.value_count); + } + + #[test] + fn memory_calls_and_division_guards_are_not_removable() { + let program = frontend::parse( + "fn quotient(x: i64, y: i64) -> i64 { let z: i64 = x / y; print_i64(z); return z; } + fn main() -> i64 { return quotient(10, 2); }", + ) + .unwrap(); + let module = emit_structured(&program).unwrap(); + let instructions: Vec<_> = module + .functions + .iter() + .flat_map(|function| &function.blocks) + .flat_map(|block| &block.instructions) + .collect(); + for marker in [ + "alloca", + "load", + "store", + "call", + "unreachable", + "ret", + "br ", + ] { + let matching: Vec<_> = instructions + .iter() + .filter(|instruction| instruction.text.contains(marker)) + .collect(); + assert!(!matching.is_empty(), "missing {marker}"); + assert!( + matching.iter().all(|instruction| !instruction.pure), + "unsafe {marker}" + ); + } + let division = instructions + .iter() + .find(|instruction| instruction.text.contains(" = sdiv ")) + .unwrap(); + assert!(division.pure); + assert_eq!(division.uses.len(), 2); + } +} +impl Ty { + fn llvm(self) -> &'static str { + match self { + Self::I64 => "i64", + Self::Bool => "i1", + } + } + fn from_ast(value: &Type) -> Self { + match value { + Type::I64 => Self::I64, + Type::Bool => Self::Bool, + } + } +} +#[derive(Clone)] +struct Signature { + params: Vec, + result: Ty, +} +#[derive(Clone)] +struct Binding { + pointer: String, + pointer_id: usize, + ty: Ty, + mutable: bool, +} +struct Value { + text: String, + ty: Ty, + id: Option, +} + +pub fn emit(program: &Program) -> Result { + Ok(emit_structured(program)?.text) +} + +pub fn emit_structured(program: &Program) -> Result { + let mut signatures = HashMap::new(); + for function in &program.functions { + if function.name == "print_i64" { + return Err("print_i64 is a reserved builtin".into()); + } + if !valid_name(&function.name) { + return Err(format!("Invalid function name: {}", function.name)); + } + let mut names = HashSet::new(); + for (name, _) in &function.params { + if !names.insert(name) { + return Err(format!("Duplicate parameter {name} in {}", function.name)); + } + } + let signature = Signature { + params: function + .params + .iter() + .map(|(_, ty)| Ty::from_ast(ty)) + .collect(), + result: Ty::from_ast(&function.return_type), + }; + if signatures + .insert(function.name.clone(), signature) + .is_some() + { + return Err(format!("Duplicate function: {}", function.name)); + } + } + let main = signatures + .get("main") + .ok_or("Program requires a main function")?; + if !main.params.is_empty() { + return Err("main must take no parameters".into()); + } + let preamble = String::from( + "; Experimental Rust subset compiler: target-neutral LLVM IR\n\ + @.format = private unnamed_addr constant [6 x i8] c\"%lld\\0A\\00\", align 1\n\ + declare i32 @printf(ptr, ...)\n\ + declare void @llvm.trap() cold noreturn nounwind\n\n", + ); + let mut functions = Vec::new(); + for function in &program.functions { + functions.push(FunctionEmitter::new(&signatures, function).emit(function)?); + } + let mut instructions = vec![InstructionIR { + text: format!(" %result = call {} @r_main()", main.result.llvm()), + definition: Some(0), + uses: vec![], + pure: false, + is_phi: false, + }]; + let printable = if main.result == Ty::Bool { + instructions.push(InstructionIR { + text: " %printable = zext i1 %result to i64".into(), + definition: Some(1), + uses: vec![0], + pure: true, + is_phi: false, + }); + ("%printable", 1) + } else { + ("%result", 0) + }; + let printed_id = printable.1 + 1; + instructions.push(InstructionIR { + text: format!( + " %printed = call i32 (ptr, ...) @printf(ptr @.format, i64 {})", + printable.0 + ), + definition: Some(printed_id), + uses: vec![printable.1], + pure: false, + is_phi: false, + }); + instructions.push(InstructionIR { + text: " ret i32 0".into(), + definition: None, + uses: vec![], + pure: false, + is_phi: false, + }); + functions.push(FunctionIR { + name: "main".into(), + header: "define i32 @main() {\n".into(), + footer: "}\n".into(), + value_count: printed_id + 1, + blocks: vec![BlockIR { + name: "entry".into(), + instructions, + successors: vec![], + }], + phi_edges: vec![], + }); + let mut module = ModuleIR { + text: String::new(), + preamble, + functions, + postamble: String::new(), + }; + module.text = module.render(); + Ok(module) +} + +fn operand_ids(values: &[&Value]) -> Vec { + values.iter().filter_map(|value| value.id).collect() +} + +fn valid_name(name: &str) -> bool { + let mut bytes = name.bytes(); + matches!(bytes.next(), Some(b'a'..=b'z' | b'A'..=b'Z' | b'_')) + && bytes.all(|c| c.is_ascii_alphanumeric() || c == b'_') +} + +struct FunctionEmitter<'a> { + signatures: &'a HashMap, + scopes: Vec>, + allocas: Vec, + blocks: Vec, + successor_labels: Vec<(usize, String)>, + phi_labels: Vec<(String, String, Vec)>, + next_value: usize, + parameter_count: usize, + next_label: usize, + current_label: String, + terminated: bool, + result: Ty, + name: String, +} + +impl<'a> FunctionEmitter<'a> { + fn new(signatures: &'a HashMap, function: &Function) -> Self { + Self { + signatures, + scopes: vec![HashMap::new()], + allocas: vec![], + blocks: vec![BlockIR { + name: "entry".into(), + instructions: vec![], + successors: vec![], + }], + successor_labels: vec![], + phi_labels: vec![], + next_value: 0, + parameter_count: function.params.len(), + next_label: 0, + current_label: "entry".into(), + terminated: false, + result: Ty::from_ast(&function.return_type), + name: function.name.clone(), + } + } + fn emit(mut self, function: &Function) -> Result { + let mut parameters = Vec::new(); + for (index, (name, ty)) in function.params.iter().enumerate() { + let ty = Ty::from_ast(ty); + parameters.push(format!("{} %arg{index}", ty.llvm())); + let (pointer, pointer_id) = self.allocate(ty); + self.line( + format!("store {} %arg{index}, ptr {pointer}", ty.llvm()), + vec![index, pointer_id], + ); + self.scopes[0].insert( + name.clone(), + Binding { + pointer, + pointer_id, + ty, + mutable: false, + }, + ); + } + self.statements(&function.body, true)?; + if !self.terminated { + return Err(format!( + "Function {} can reach its end without returning", + self.name + )); + } + let header = format!( + "define {} @r_{}({}) {{\n", + self.result.llvm(), + function.name, + parameters.join(", ") + ); + self.allocas.append(&mut self.blocks[0].instructions); + self.blocks[0].instructions = self.allocas; + let labels: HashMap<_, _> = self + .blocks + .iter() + .enumerate() + .map(|(index, block)| (block.name.as_str(), index)) + .collect(); + let mut successors = vec![Vec::new(); self.blocks.len()]; + for (from, label) in &self.successor_labels { + let to = *labels + .get(label.as_str()) + .ok_or_else(|| format!("Missing basic block {label}"))?; + if !successors[*from].contains(&to) { + successors[*from].push(to); + } + } + let mut phi_edges = Vec::new(); + for (from, to, values) in self.phi_labels { + let from = *labels + .get(from.as_str()) + .ok_or_else(|| format!("Missing phi predecessor {from}"))?; + let to = *labels + .get(to.as_str()) + .ok_or_else(|| format!("Missing phi target {to}"))?; + phi_edges.push(PhiEdge { from, to, values }); + } + for (block, edges) in self.blocks.iter_mut().zip(successors) { + block.successors = edges; + } + Ok(FunctionIR { + name: format!("r_{}", function.name), + header, + footer: "}\n\n".into(), + value_count: self.parameter_count + self.next_value, + blocks: self.blocks, + phi_edges, + }) + } + fn temporary(&mut self) -> (String, usize) { + let name = format!("%v{}", self.next_value); + let id = self.parameter_count + self.next_value; + self.next_value += 1; + (name, id) + } + fn label(&mut self) -> String { + let name = format!("bb{}", self.next_label); + self.next_label += 1; + name + } + fn line(&mut self, line: String, uses: Vec) { + self.blocks + .last_mut() + .unwrap() + .instructions + .push(InstructionIR { + text: format!(" {line}"), + definition: None, + uses, + pure: false, + is_phi: false, + }); + } + fn start_block(&mut self, label: &str) { + self.blocks.push(BlockIR { + name: label.into(), + instructions: vec![], + successors: vec![], + }); + self.current_label = label.into(); + self.terminated = false; + } + fn branch(&mut self, label: &str) { + self.successor_labels + .push((self.blocks.len() - 1, label.into())); + self.line(format!("br label %{label}"), vec![]); + self.terminated = true; + } + fn conditional(&mut self, condition: &Value, yes: &str, no: &str) { + self.successor_labels + .push((self.blocks.len() - 1, yes.into())); + self.successor_labels + .push((self.blocks.len() - 1, no.into())); + self.line( + format!("br i1 {}, label %{yes}, label %{no}", condition.text), + operand_ids(&[condition]), + ); + self.terminated = true; + } + fn instruction(&mut self, ty: Ty, instruction: String, uses: Vec, pure: bool) -> Value { + let (text, id) = self.temporary(); + self.blocks + .last_mut() + .unwrap() + .instructions + .push(InstructionIR { + text: format!(" {text} = {instruction}"), + definition: Some(id), + uses, + pure, + is_phi: false, + }); + Value { + text, + ty, + id: Some(id), + } + } + fn allocate(&mut self, ty: Ty) -> (String, usize) { + let (pointer, id) = self.temporary(); + self.allocas.push(InstructionIR { + text: format!( + " {pointer} = alloca {}, align {}", + ty.llvm(), + if ty == Ty::I64 { 8 } else { 1 } + ), + definition: Some(id), + uses: vec![], + pure: false, + is_phi: false, + }); + (pointer, id) + } + fn require(&self, actual: Ty, expected: Ty, context: &str) -> Result<(), String> { + if actual == expected { + Ok(()) + } else { + Err(format!( + "Type mismatch in {context}: expected {expected:?}, got {actual:?}" + )) + } + } + fn lookup(&self, name: &str) -> Result { + self.scopes + .iter() + .rev() + .find_map(|scope| scope.get(name).cloned()) + .ok_or_else(|| format!("Unknown variable: {name}")) + } + fn statements(&mut self, statements: &[Stmt], function_body: bool) -> Result<(), String> { + for (index, statement) in statements.iter().enumerate() { + if self.terminated { + break; + } + if function_body && index + 1 == statements.len() { + if let Stmt::Expr(expr) = statement { + if !matches!(expr, Expr::Call { name, .. } if name == "print_i64") { + let value = self.expression(expr)?; + self.return_value(value)?; + continue; + } + } + } + self.statement(statement)?; + } + Ok(()) + } + fn nested(&mut self, statements: &[Stmt]) -> Result<(), String> { + self.scopes.push(HashMap::new()); + let result = self.statements(statements, false); + self.scopes.pop(); + result + } + fn return_value(&mut self, value: Value) -> Result<(), String> { + self.require(value.ty, self.result, "return")?; + self.line( + format!("ret {} {}", value.ty.llvm(), value.text), + operand_ids(&[&value]), + ); + self.terminated = true; + Ok(()) + } + fn statement(&mut self, statement: &Stmt) -> Result<(), String> { + match statement { + Stmt::Let { + name, + ty, + mutable, + value, + } => { + let value = self.expression(value)?; + if let Some(annotation) = ty { + self.require(value.ty, Ty::from_ast(annotation), "let initializer")?; + } + let (pointer, pointer_id) = self.allocate(value.ty); + let mut uses = operand_ids(&[&value]); + uses.push(pointer_id); + self.line( + format!("store {} {}, ptr {pointer}", value.ty.llvm(), value.text), + uses, + ); + self.scopes.last_mut().unwrap().insert( + name.clone(), + Binding { + pointer, + pointer_id, + ty: value.ty, + mutable: *mutable, + }, + ); + } + Stmt::Assign { name, value } => { + let binding = self.lookup(name)?; + if !binding.mutable { + return Err(format!("Cannot assign to immutable variable: {name}")); + } + let value = self.expression(value)?; + self.require(value.ty, binding.ty, "assignment")?; + self.line( + format!( + "store {} {}, ptr {}", + value.ty.llvm(), + value.text, + binding.pointer + ), + { + let mut uses = operand_ids(&[&value]); + uses.push(binding.pointer_id); + uses + }, + ); + } + Stmt::Return(expr) => { + let value = self.expression(expr)?; + self.return_value(value)?; + } + Stmt::Expr(Expr::Call { name, args }) if name == "print_i64" => { + if args.len() != 1 { + return Err("print_i64 requires one i64 argument".into()); + } + let value = self.expression(&args[0])?; + self.require(value.ty, Ty::I64, "print_i64 argument")?; + self.line( + format!( + "call i32 (ptr, ...) @printf(ptr @.format, i64 {})", + value.text + ), + operand_ids(&[&value]), + ); + } + Stmt::Expr(expr) => { + self.expression(expr)?; + } + Stmt::If { + condition, + then_body, + else_body, + } => { + let condition = self.expression(condition)?; + self.require(condition.ty, Ty::Bool, "if condition")?; + let yes = self.label(); + let no = self.label(); + let end = self.label(); + self.conditional(&condition, &yes, &no); + self.start_block(&yes); + self.nested(then_body)?; + let yes_falls = !self.terminated; + if yes_falls { + self.branch(&end); + } + self.start_block(&no); + self.nested(else_body)?; + let no_falls = !self.terminated; + if no_falls { + self.branch(&end); + } + if yes_falls || no_falls { + self.start_block(&end); + } + } + Stmt::While { condition, body } => { + let test = self.label(); + let loop_body = self.label(); + let end = self.label(); + self.branch(&test); + self.start_block(&test); + let condition = self.expression(condition)?; + self.require(condition.ty, Ty::Bool, "while condition")?; + self.conditional(&condition, &loop_body, &end); + self.start_block(&loop_body); + self.nested(body)?; + if !self.terminated { + self.branch(&test); + } + self.start_block(&end); + } + } + Ok(()) + } + fn expression(&mut self, expression: &Expr) -> Result { + match expression { + Expr::Int(number) => Ok(Value { + text: number.to_string(), + ty: Ty::I64, + id: None, + }), + Expr::Bool(value) => Ok(Value { + text: value.to_string(), + ty: Ty::Bool, + id: None, + }), + Expr::Var(name) => { + let binding = self.lookup(name)?; + Ok(self.instruction( + binding.ty, + format!("load {}, ptr {}", binding.ty.llvm(), binding.pointer), + vec![binding.pointer_id], + false, + )) + } + Expr::Call { name, args } => { + if name == "print_i64" { + return Err("print_i64 can only be used as a statement".into()); + } + let signature = self + .signatures + .get(name) + .cloned() + .ok_or_else(|| format!("Unknown function: {name}"))?; + if args.len() != signature.params.len() { + return Err(format!("Wrong argument count for {name}")); + } + let mut arguments = Vec::new(); + let mut uses = Vec::new(); + for (expression, expected) in args.iter().zip(signature.params.iter()) { + let value = self.expression(expression)?; + self.require(value.ty, *expected, "call argument")?; + arguments.push(format!("{} {}", value.ty.llvm(), value.text)); + uses.extend(value.id); + } + Ok(self.instruction( + signature.result, + format!( + "call {} @r_{name}({})", + signature.result.llvm(), + arguments.join(", ") + ), + uses, + false, + )) + } + Expr::Unary { op, value } => { + let value = self.expression(value)?; + match op.as_str() { + "-" => { + self.require(value.ty, Ty::I64, "unary minus")?; + Ok(self.instruction( + Ty::I64, + format!("sub i64 0, {}", value.text), + operand_ids(&[&value]), + true, + )) + } + "!" => Ok(self.instruction( + value.ty, + format!( + "xor {} {}, {}", + value.ty.llvm(), + value.text, + if value.ty == Ty::Bool { "true" } else { "-1" } + ), + operand_ids(&[&value]), + true, + )), + _ => Err(format!("Unsupported unary operator: {op}")), + } + } + Expr::Binary { op, left, right } if op == "&&" || op == "||" => { + self.short_circuit(op, left, right) + } + Expr::Binary { op, left, right } => { + let left = self.expression(left)?; + let right = self.expression(right)?; + self.require(right.ty, left.ty, "binary operands")?; + match op.as_str() { + "==" | "!=" => Ok(self.instruction( + Ty::Bool, + format!( + "icmp {} {} {}, {}", + if op == "==" { "eq" } else { "ne" }, + left.ty.llvm(), + left.text, + right.text + ), + operand_ids(&[&left, &right]), + true, + )), + "<" | "<=" | ">" | ">=" => { + self.require(left.ty, Ty::I64, "comparison")?; + let predicate = match op.as_str() { + "<" => "slt", + "<=" => "sle", + ">" => "sgt", + _ => "sge", + }; + Ok(self.instruction( + Ty::Bool, + format!("icmp {predicate} i64 {}, {}", left.text, right.text), + operand_ids(&[&left, &right]), + true, + )) + } + "&" | "|" | "^" => { + let instruction = match op.as_str() { + "&" => "and", + "|" => "or", + _ => "xor", + }; + Ok(self.instruction( + left.ty, + format!( + "{instruction} {} {}, {}", + left.ty.llvm(), + left.text, + right.text + ), + operand_ids(&[&left, &right]), + true, + )) + } + "+" | "-" | "*" | "/" | "%" | "<<" | ">>" => { + self.require(left.ty, Ty::I64, "integer arithmetic")?; + if op == "/" || op == "%" { + self.guard_division(&left, &right); + } + let instruction = match op.as_str() { + "+" => "add", + "-" => "sub", + "*" => "mul", + "/" => "sdiv", + "%" => "srem", + "<<" => "shl", + _ => "ashr", + }; + let right = if op == "<<" || op == ">>" { + self.instruction( + Ty::I64, + format!("and i64 {}, 63", right.text), + operand_ids(&[&right]), + true, + ) + } else { + right + }; + Ok(self.instruction( + Ty::I64, + format!("{instruction} i64 {}, {}", left.text, right.text), + operand_ids(&[&left, &right]), + true, + )) + } + _ => Err(format!("Unsupported binary operator: {op}")), + } + } + } + } + fn short_circuit(&mut self, op: &str, left: &Expr, right: &Expr) -> Result { + let left = self.expression(left)?; + self.require(left.ty, Ty::Bool, "boolean operand")?; + let left_block = self.current_label.clone(); + let rhs = self.label(); + let end = self.label(); + if op == "&&" { + self.conditional(&left, &rhs, &end); + } else { + self.conditional(&left, &end, &rhs); + } + self.start_block(&rhs); + let right = self.expression(right)?; + self.require(right.ty, Ty::Bool, "boolean operand")?; + let right_block = self.current_label.clone(); + self.branch(&end); + self.start_block(&end); + let constant = if op == "&&" { "false" } else { "true" }; + // A phi reads its incoming value on the predecessor edge, not in the + // merge block. The constant incoming value needs no liveness bit. + self.phi_labels + .push((right_block.clone(), end, operand_ids(&[&right]))); + let value = self.instruction( + Ty::Bool, + format!( + "phi i1 [{constant}, %{left_block}], [{}, %{right_block}]", + right.text + ), + vec![], + true, + ); + self.blocks + .last_mut() + .unwrap() + .instructions + .last_mut() + .unwrap() + .is_phi = true; + Ok(value) + } + fn guard_division(&mut self, left: &Value, right: &Value) { + let zero = self.instruction( + Ty::Bool, + format!("icmp eq i64 {}, 0", right.text), + operand_ids(&[right]), + true, + ); + let minimum = self.instruction( + Ty::Bool, + format!("icmp eq i64 {}, -9223372036854775808", left.text), + operand_ids(&[left]), + true, + ); + let negative_one = self.instruction( + Ty::Bool, + format!("icmp eq i64 {}, -1", right.text), + operand_ids(&[right]), + true, + ); + let overflow = self.instruction( + Ty::Bool, + format!("and i1 {}, {}", minimum.text, negative_one.text), + operand_ids(&[&minimum, &negative_one]), + true, + ); + let invalid = self.instruction( + Ty::Bool, + format!("or i1 {}, {}", zero.text, overflow.text), + operand_ids(&[&zero, &overflow]), + true, + ); + let trap = self.label(); + let valid = self.label(); + self.conditional(&invalid, &trap, &valid); + self.start_block(&trap); + self.line("call void @llvm.trap()".into(), vec![]); + self.line("unreachable".into(), vec![]); + self.terminated = true; + self.start_block(&valid); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/machine_codegen.rs b/experiments/gpu-rust-compiler/compiler/src/machine_codegen.rs new file mode 100644 index 00000000000..0d9d6953d33 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/machine_codegen.rs @@ -0,0 +1,522 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::codegen_service::{ + CallingConvention, Emission, ExecutionRecord, FlatBatch, FunctionEmission, Opcode, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MachineTarget { + X86_64, + Aarch64, +} + +impl MachineTarget { + pub fn number(self) -> u32 { + match self { + Self::X86_64 => 0, + Self::Aarch64 => 1, + } + } + pub fn host() -> Result { + if cfg!(target_arch = "x86_64") { + Ok(Self::X86_64) + } else if cfg!(target_arch = "aarch64") { + Ok(Self::Aarch64) + } else { + Err("native emission supports x86-64 and AArch64 only".into()) + } + } + pub fn host_triple(self) -> &'static str { + match (self, cfg!(target_os = "macos")) { + (Self::X86_64, true) => "x86_64-apple-darwin", + (Self::X86_64, false) => "x86_64-unknown-linux-gnu", + (Self::Aarch64, true) => "aarch64-apple-darwin", + (Self::Aarch64, false) => "aarch64-unknown-linux-gnu", + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct PackedInstruction { + pub opcode: u32, + pub result: u32, + pub a: u32, + pub b: u32, + pub immediate: i64, +} + +pub const ARGUMENT: u32 = 0; +pub const CONSTANT: u32 = 1; +pub const COPY: u32 = 2; +pub const ADD: u32 = 3; +pub const SUB: u32 = 4; +pub const MUL: u32 = 5; +pub const RETURN: u32 = 6; + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct PackedFunction { + pub start: u32, + pub count: u32, + pub values: u32, + pub symbol: u32, +} + +pub struct Packet { + pub target: MachineTarget, + pub functions: Vec, + pub instructions: Vec, + pub output_capacity: usize, +} + +pub fn pack(batch: &FlatBatch, target: MachineTarget) -> Result { + let supported = match target { + MachineTarget::X86_64 => matches!( + batch.identity().target_abi.as_str(), + "x86_64-unknown-linux-gnu" | "x86_64-apple-darwin" + ), + MachineTarget::Aarch64 => matches!( + batch.identity().target_abi.as_str(), + "aarch64-unknown-linux-gnu" | "aarch64-apple-darwin" + ), + }; + if !supported { + return Err("batch target identity does not match machine-code target".into()); + } + let mut functions = Vec::with_capacity(batch.functions().len()); + let mut output_capacity = 0usize; + for f in batch.functions() { + let abi = &batch.abis()[f.abi as usize]; + let compatible = match target { + MachineTarget::X86_64 => abi.calling_convention == CallingConvention::SystemV, + MachineTarget::Aarch64 => matches!( + abi.calling_convention, + CallingConvention::AppleAarch64 | CallingConvention::SystemV + ), + }; + if !compatible || !abi.returns_i64 || abi.parameter_count > 1 { + return Err( + "scalar emission requires matching C ABI and at most one i64 argument/result" + .into(), + ); + } + if f.value_count == 0 || f.value_count > 256 { + return Err("scalar emission supports 1..256 value slots".into()); + } + output_capacity = output_capacity + .checked_add(12 + 25 * f.instruction_count as usize) + .ok_or("output size overflow")?; + functions.push(PackedFunction { + start: f.instruction_start, + count: f.instruction_count, + values: f.value_count, + symbol: f.symbol, + }); + } + if output_capacity > 512 * 1024 * 1024 { + return Err("scalar emission output exceeds 512 MiB budget".into()); + } + let instructions = batch + .instructions() + .iter() + .map(|i| PackedInstruction { + opcode: match i.opcode { + Opcode::Argument => ARGUMENT, + Opcode::Const => CONSTANT, + Opcode::Copy => COPY, + Opcode::Add => ADD, + Opcode::Sub => SUB, + Opcode::Mul => MUL, + Opcode::Return => RETURN, + }, + result: i.result.unwrap_or(u32::MAX), + a: i.operands[0], + b: i.operands[1], + immediate: i.immediate, + }) + .collect(); + Ok(Packet { + target, + functions, + instructions, + output_capacity, + }) +} + +fn validate(values: u32, instructions: &[PackedInstruction]) -> Result<(), String> { + if values == 0 || values > 256 || instructions.is_empty() { + return Err("invalid scalar frame or empty function".into()); + } + let mut definitions = vec![false; values as usize]; + for (pos, i) in instructions.iter().enumerate() { + if (i.opcode == RETURN) != (pos + 1 == instructions.len()) { + return Err("function must end with its only return".into()); + } + let operands = match i.opcode { + ARGUMENT | CONSTANT => 0, + COPY | RETURN => 1, + ADD | SUB | MUL => 2, + _ => return Err("unsupported opcode".into()), + }; + for value in [i.a, i.b].into_iter().take(operands) { + if value >= values || !definitions[value as usize] { + return Err("undefined operand".into()); + } + } + if i.opcode != RETURN { + if i.result >= values || definitions[i.result as usize] { + return Err("invalid or repeated definition".into()); + } + definitions[i.result as usize] = true; + } + if i.opcode == ARGUMENT && i.immediate != 0 { + return Err("only argument zero is supported".into()); + } + } + Ok(()) +} + +pub fn emit_function( + target: MachineTarget, + values: u32, + instructions: &[PackedInstruction], +) -> Result, String> { + validate(values, instructions)?; + Ok(emit_validated(target, values, instructions)) +} + +fn emit_validated( + target: MachineTarget, + values: u32, + instructions: &[PackedInstruction], +) -> Vec { + let frame = (values * 8 + 15) & !15; + let mut code = Vec::new(); + if target == MachineTarget::X86_64 { + code.extend_from_slice(&[0x55, 0x48, 0x89, 0xe5, 0x48, 0x81, 0xec]); + code.extend_from_slice(&frame.to_le_bytes()); + fn mem(out: &mut Vec, bytes: &[u8], slot: u32) { + out.extend_from_slice(bytes); + out.extend_from_slice(&(-8i32 * (slot as i32 + 1)).to_le_bytes()); + } + for i in instructions { + match i.opcode { + ARGUMENT => mem(&mut code, &[0x48, 0x89, 0xbd], i.result), + CONSTANT => { + code.extend_from_slice(&[0x48, 0xb8]); + code.extend_from_slice(&i.immediate.to_le_bytes()); + mem(&mut code, &[0x48, 0x89, 0x85], i.result); + } + COPY => { + mem(&mut code, &[0x48, 0x8b, 0x85], i.a); + mem(&mut code, &[0x48, 0x89, 0x85], i.result); + } + ADD | SUB | MUL => { + mem(&mut code, &[0x48, 0x8b, 0x85], i.a); + mem(&mut code, &[0x4c, 0x8b, 0x95], i.b); + code.extend_from_slice(match i.opcode { + ADD => &[0x4c, 0x01, 0xd0][..], + SUB => &[0x4c, 0x29, 0xd0][..], + _ => &[0x49, 0x0f, 0xaf, 0xc2][..], + }); + mem(&mut code, &[0x48, 0x89, 0x85], i.result); + } + RETURN => { + mem(&mut code, &[0x48, 0x8b, 0x85], i.a); + code.extend_from_slice(&[0xc9, 0xc3]); + } + _ => unreachable!(), + } + } + } else { + fn word(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_le_bytes()); + } + word(&mut code, 0xa9bf7bfd); + word(&mut code, 0x910003fd); + word(&mut code, 0xd10003ff | (frame << 10)); + for i in instructions { + match i.opcode { + ARGUMENT => word(&mut code, 0xf90003e0 | (i.result << 10)), + CONSTANT => { + let value = i.immediate as u64; + for part in 0..4 { + word( + &mut code, + (if part == 0 { 0xd2800009 } else { 0xf2800009 }) + | (part << 21) + | (((value >> (part * 16)) as u32 & 0xffff) << 5), + ); + } + word(&mut code, 0xf90003e9 | (i.result << 10)); + } + COPY => { + word(&mut code, 0xf94003e9 | (i.a << 10)); + word(&mut code, 0xf90003e9 | (i.result << 10)); + } + ADD | SUB | MUL => { + word(&mut code, 0xf94003e9 | (i.a << 10)); + word(&mut code, 0xf94003ea | (i.b << 10)); + word( + &mut code, + match i.opcode { + ADD => 0x8b0a0129, + SUB => 0xcb0a0129, + _ => 0x9b0a7d29, + }, + ); + word(&mut code, 0xf90003e9 | (i.result << 10)); + } + RETURN => { + word(&mut code, 0xf94003e0 | (i.a << 10)); + word(&mut code, 0x910003bf); + word(&mut code, 0xa8c17bfd); + word(&mut code, 0xd65f03c0); + } + _ => unreachable!(), + } + } + } + code +} + +pub fn cpu_emit(batch: &FlatBatch, target: MachineTarget) -> Result { + let packet = pack(batch, target)?; + let functions = packet + .functions + .iter() + .enumerate() + .map(|(index, f)| { + let code = emit_validated( + target, + f.values, + &packet.instructions[f.start as usize..(f.start + f.count) as usize], + ); + let unwind = if batch.abis()[batch.functions()[index].abi as usize].requires_unwind { + unwind_descriptor(target, code.len()) + } else { + Vec::new() + }; + FunctionEmission { + symbol: f.symbol, + code, + relocations: Vec::new(), + unwind, + } + }) + .collect(); + Ok(Emission { + functions, + execution: ExecutionRecord::Cpu, + }) +} + +/// Metadata describing the encoder's fixed frame. The object writer expands +/// this to platform records; it is not itself a serialized DWARF program. +pub fn unwind_descriptor(target: MachineTarget, length: usize) -> Vec { + let mut descriptor = b"GUF1".to_vec(); + descriptor.extend_from_slice(&target.number().to_le_bytes()); + descriptor.extend_from_slice(&(length as u32).to_le_bytes()); + descriptor +} + +fn dwarf_frame(target: MachineTarget, length: usize) -> (Vec, u64) { + let mut out = vec![0; 4]; + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&[1, b'z', b'R', 0, 1, 0x78]); + out.extend_from_slice(&[ + if target == MachineTarget::X86_64 { + 16 + } else { + 30 + }, + 1, + 0x1b, + ]); + out.extend_from_slice(if target == MachineTarget::X86_64 { + &[0x0c, 7, 8, 0x90, 1, 0x08, 6] + } else { + &[0x0c, 31, 0, 0x08, 30, 0x08, 29] + }); + while !out.len().is_multiple_of(4) { + out.push(0); + } + let len = (out.len() - 4) as u32; + out[..4].copy_from_slice(&len.to_le_bytes()); + let fde = out.len(); + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&((fde + 4) as u32).to_le_bytes()); + let location = out.len() as u64; + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&(length as u32).to_le_bytes()); + out.push(0); + if target == MachineTarget::X86_64 { + // push rbp; mov rbp,rsp; ...; leave; ret. CFA follows rsp at entry, + // then rbp+16, then rsp+8 after leave restores the caller's frame. + out.extend_from_slice(&[0x41, 0x0e, 16, 0x86, 2, 0x43, 0x0d, 6, 0x04]); + out.extend_from_slice(&((length - 5) as u32).to_le_bytes()); + out.extend_from_slice(&[0x0c, 7, 8, 0xc6]); + } else { + // stp fp,lr,[sp,-16]!; mov fp,sp; ...; ldp fp,lr,[sp],16; ret. + out.extend_from_slice(&[0x44, 0x0e, 16, 0x9d, 2, 0x9e, 1, 0x44, 0x0d, 29, 0x04]); + out.extend_from_slice(&((length - 12) as u32).to_le_bytes()); + out.extend_from_slice(&[0x0c, 31, 0, 0xdd, 0xde]); + } + while !out.len().is_multiple_of(4) { + out.push(0); + } + let len = (out.len() - fde - 4) as u32; + out[fde..fde + 4].copy_from_slice(&len.to_le_bytes()); + (out, location) +} + +pub fn write_object( + batch: &FlatBatch, + target: MachineTarget, + emission: &Emission, +) -> Result, String> { + use object::write::{Object, StandardSection, Symbol, SymbolSection}; + let format = if batch.identity().target_abi.contains("apple") { + object::BinaryFormat::MachO + } else { + object::BinaryFormat::Elf + }; + let architecture = match target { + MachineTarget::X86_64 => object::Architecture::X86_64, + MachineTarget::Aarch64 => object::Architecture::Aarch64, + }; + let mut object = Object::new(format, architecture, object::Endianness::Little); + let text = object.section_id(StandardSection::Text); + if emission.functions.len() != batch.functions().len() { + return Err("incomplete object emission".into()); + } + let mut functions: Vec<_> = emission.functions.iter().collect(); + functions.sort_by_key(|f| f.symbol); + let mut unwind_section = None; + let mut macho_unwind = None; + for f in functions { + if !f.relocations.is_empty() { + return Err("scalar object writer does not support code relocations".into()); + } + if !f.unwind.is_empty() && f.unwind != unwind_descriptor(target, f.code.len()) { + return Err("unwind descriptor does not match fixed-frame encoder".into()); + } + let name = &batch + .symbols() + .get(f.symbol as usize) + .ok_or("unknown symbol")? + .name; + if !name.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'_') { + return Err("fixture symbol must be a C identifier".into()); + } + let offset = object.append_section_data(text, &f.code, 16); + let symbol = object.add_symbol(Symbol { + name: name.as_bytes().to_vec(), + value: offset, + size: f.code.len() as u64, + kind: object::SymbolKind::Text, + scope: object::SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(text), + flags: object::SymbolFlags::None, + }); + if !f.unwind.is_empty() { + if format == object::BinaryFormat::MachO { + let sections = macho_unwind + .get_or_insert_with(|| crate::macho_unwind::Sections::new(&mut object)); + sections.append(&mut object, target, symbol, f.code.len())?; + continue; + } + let section = *unwind_section.get_or_insert_with(|| { + object.add_section( + Vec::new(), + b".eh_frame".to_vec(), + object::SectionKind::ReadOnlyData, + ) + }); + let (data, location) = dwarf_frame(target, f.code.len()); + let base = object.append_section_data(section, &data, 8); + object + .add_relocation( + section, + object::write::Relocation { + offset: base + location, + symbol, + addend: 0, + flags: object::RelocationFlags::Generic { + kind: object::RelocationKind::Relative, + encoding: object::RelocationEncoding::Generic, + size: 32, + }, + }, + ) + .map_err(|e| e.to_string())?; + } + } + if let Some(sections) = macho_unwind { + sections.finish(&mut object)?; + } + if format == object::BinaryFormat::Elf { + object.add_section( + Vec::new(), + b".note.GNU-stack".to_vec(), + object::SectionKind::Other, + ); + } + object.write().map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arm_leaf_constant_has_native_return_and_fixed_frame() { + let input = [ + PackedInstruction { + opcode: CONSTANT, + result: 0, + a: 0, + b: 0, + immediate: 42, + }, + PackedInstruction { + opcode: RETURN, + result: u32::MAX, + a: 0, + b: 0, + immediate: 0, + }, + ]; + let code = emit_function(MachineTarget::Aarch64, 1, &input).unwrap(); + assert_eq!(code.len(), 48); + assert_eq!(&code[..4], &0xa9bf7bfdu32.to_le_bytes()); + assert_eq!(&code[code.len() - 4..], &0xd65f03c0u32.to_le_bytes()); + } + + #[test] + fn x86_leaf_constant_has_native_return_and_fixed_frame() { + let input = [ + PackedInstruction { + opcode: CONSTANT, + result: 0, + a: 0, + b: 0, + immediate: 42, + }, + PackedInstruction { + opcode: RETURN, + result: u32::MAX, + a: 0, + b: 0, + immediate: 0, + }, + ]; + let code = emit_function(MachineTarget::X86_64, 1, &input).unwrap(); + assert_eq!(code.len(), 37); + assert_eq!(&code[..4], &[0x55, 0x48, 0x89, 0xe5]); + assert_eq!(&code[code.len() - 2..], &[0xc9, 0xc3]); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/macho_unwind.rs b/experiments/gpu-rust-compiler/compiler/src/macho_unwind.rs new file mode 100644 index 00000000000..491eaea2427 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/macho_unwind.rs @@ -0,0 +1,290 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::machine_codegen::MachineTarget; +use object::write::{Object, SectionId, SymbolId}; + +pub struct Sections { + frame: SectionId, + compact: SectionId, + pairs: Vec<(u64, SymbolId, SymbolId, MachineTarget)>, +} + +impl Sections { + pub fn new(object: &mut Object<'_>) -> Self { + // Each exported leaf is an independent atom. Without this flag, Apple's + // linker can give the last function a zero-sized compact unwind range. + let flags = match object.flags { + object::FileFlags::MachO { flags } => flags, + _ => 0, + }; + object.flags = object::FileFlags::MachO { + flags: flags | object::macho::MH_SUBSECTIONS_VIA_SYMBOLS, + }; + let frame = object.add_section( + b"__TEXT".to_vec(), + b"__eh_frame".to_vec(), + object::SectionKind::ReadOnlyData, + ); + object.section_mut(frame).flags = object::SectionFlags::MachO { + flags: object::macho::S_COALESCED + | object::macho::S_ATTR_NO_TOC + | object::macho::S_ATTR_STRIP_STATIC_SYMS + | object::macho::S_ATTR_LIVE_SUPPORT, + }; + let compact = object.add_section( + b"__LD".to_vec(), + b"__compact_unwind".to_vec(), + object::SectionKind::ReadOnlyData, + ); + object.section_mut(compact).flags = object::SectionFlags::MachO { + flags: object::macho::S_ATTR_DEBUG, + }; + Self { + frame, + compact, + pairs: Vec::new(), + } + } + + pub fn append( + &mut self, + object: &mut Object<'_>, + target: MachineTarget, + symbol: SymbolId, + length: usize, + ) -> Result<(), String> { + if length < 20 + || length > u32::MAX as usize + || (target == MachineTarget::Aarch64 && (length < 28 || !length.is_multiple_of(4))) + { + return Err("invalid fixed-frame code length for unwind records".into()); + } + let (frame, pointer) = dwarf_record(target, length); + let offset = object.append_section_data(self.frame, &frame, 8); + let location = offset + pointer as u64; + let anchor = object.add_symbol(object::write::Symbol { + name: format!("L_gpu_fde_{location}").into_bytes(), + value: location, + size: 0, + kind: object::SymbolKind::Data, + scope: object::SymbolScope::Compilation, + weak: false, + section: object::write::SymbolSection::Section(self.frame), + flags: object::SymbolFlags::None, + }); + self.pairs.push((location, anchor, symbol, target)); + let mut compact = vec![0; 8]; + compact.extend_from_slice(&(length as u32).to_le_bytes()); + compact.extend_from_slice( + &(if target == MachineTarget::Aarch64 { + 0x0300_0000u32 + } else { + 0x0400_0000u32 + }) + .to_le_bytes(), + ); + compact.extend_from_slice(&[0; 16]); + let offset = object.append_section_data(self.compact, &compact, 8); + object + .add_relocation( + self.compact, + object::write::Relocation { + offset, + symbol, + addend: 0, + flags: object::RelocationFlags::Generic { + kind: object::RelocationKind::Absolute, + encoding: object::RelocationEncoding::Generic, + size: 64, + }, + }, + ) + .map_err(|e| e.to_string())?; + Ok(()) + } + + pub fn finish(self, object: &mut Object<'_>) -> Result<(), String> { + // object 0.37 reverses ascending relocation vectors wholesale. Descending + // groups preserve the required SUBTRACTOR followed by UNSIGNED pair. + for (offset, anchor, symbol, target) in self.pairs.into_iter().rev() { + let (subtractor, unsigned) = if target == MachineTarget::Aarch64 { + ( + object::macho::ARM64_RELOC_SUBTRACTOR, + object::macho::ARM64_RELOC_UNSIGNED, + ) + } else { + ( + object::macho::X86_64_RELOC_SUBTRACTOR, + object::macho::X86_64_RELOC_UNSIGNED, + ) + }; + for (r_type, symbol) in [(subtractor, anchor), (unsigned, symbol)] { + object + .add_relocation( + self.frame, + object::write::Relocation { + offset, + symbol, + addend: 0, + flags: object::RelocationFlags::MachO { + r_type, + r_pcrel: false, + r_length: 3, + }, + }, + ) + .map_err(|e| e.to_string())?; + } + } + Ok(()) + } +} + +fn dwarf_record(target: MachineTarget, length: usize) -> (Vec, usize) { + let mut out = vec![0; 4]; + out.extend_from_slice(&0u32.to_le_bytes()); + // DWARF32 CIE v1, zR augmentation, code alignment 1/data alignment -8. + // Mach-O ld requires PC-relative target-width pointers (DW_EH_PE_pcrel). + out.extend_from_slice(&[ + 1, + b'z', + b'R', + 0, + 1, + 0x78, + if target == MachineTarget::X86_64 { + 16 + } else { + 30 + }, + 1, + 0x10, + ]); + out.extend_from_slice(if target == MachineTarget::X86_64 { + &[0x0c, 7, 8, 0x90, 1, 0x08, 6] + } else { + &[0x0c, 31, 0, 0x08, 30, 0x08, 29] + }); + while !out.len().is_multiple_of(8) { + out.push(0); + } + let size = (out.len() - 4) as u32; + out[..4].copy_from_slice(&size.to_le_bytes()); + let fde = out.len(); + out.extend_from_slice(&0u32.to_le_bytes()); + out.extend_from_slice(&((fde + 4) as u32).to_le_bytes()); + let pointer = out.len(); + out.extend_from_slice(&0u64.to_le_bytes()); + out.extend_from_slice(&(length as u64).to_le_bytes()); + out.push(0); + if target == MachineTarget::X86_64 { + out.extend_from_slice(&[0x41, 0x0e, 16, 0x86, 2, 0x43, 0x0d, 6, 0x04]); + out.extend_from_slice(&((length - 5) as u32).to_le_bytes()); + out.extend_from_slice(&[0x0c, 7, 8, 0xc6]); + } else { + out.extend_from_slice(&[0x44, 0x0e, 16, 0x9d, 2, 0x9e, 1, 0x44, 0x0d, 29, 0x04]); + out.extend_from_slice(&((length - 12) as u32).to_le_bytes()); + out.extend_from_slice(&[0x0c, 31, 0, 0xdd, 0xde]); + } + while !out.len().is_multiple_of(8) { + out.push(0); + } + let size = (out.len() - fde - 4) as u32; + out[fde..fde + 4].copy_from_slice(&size.to_le_bytes()); + (out, pointer) +} + +#[cfg(test)] +mod tests { + use super::*; + use object::write::{StandardSection, Symbol, SymbolSection}; + use object::{Object as _, ObjectSection as _}; + + #[test] + fn fixed_frame_uses_dwarf_compact_mode_and_retains_precise_eh_frame_rows() { + for (target, architecture, encoding, length) in [ + ( + MachineTarget::Aarch64, + object::Architecture::Aarch64, + 0x0300_0000u32, + 48, + ), + ( + MachineTarget::X86_64, + object::Architecture::X86_64, + 0x0400_0000u32, + 37, + ), + ] { + let mut object = Object::new( + object::BinaryFormat::MachO, + architecture, + object::Endianness::Little, + ); + let text = object.section_id(StandardSection::Text); + let mut sections = Sections::new(&mut object); + for name in [b"async_leaf".as_slice(), b"async_leaf_second".as_slice()] { + let offset = object.append_section_data(text, &vec![0u8; length], 16); + let symbol = object.add_symbol(Symbol { + name: name.to_vec(), + value: offset, + size: length as u64, + kind: object::SymbolKind::Text, + scope: object::SymbolScope::Dynamic, + weak: false, + section: SymbolSection::Section(text), + flags: object::SymbolFlags::None, + }); + sections + .append(&mut object, target, symbol, length) + .unwrap(); + } + sections.finish(&mut object).unwrap(); + let bytes = object.write().unwrap(); + let file = object::File::parse(bytes.as_slice()).unwrap(); + assert!(matches!( + file.flags(), + object::FileFlags::MachO { flags } + if flags & object::macho::MH_SUBSECTIONS_VIA_SYMBOLS != 0 + )); + let frame = file.section_by_name("__eh_frame").unwrap(); + assert!(!frame.data().unwrap().is_empty()); + let relocations: Vec<_> = frame.relocations().collect(); + assert_eq!(relocations.len(), 4); + let (subtractor, unsigned) = if target == MachineTarget::Aarch64 { + ( + object::macho::ARM64_RELOC_SUBTRACTOR, + object::macho::ARM64_RELOC_UNSIGNED, + ) + } else { + ( + object::macho::X86_64_RELOC_SUBTRACTOR, + object::macho::X86_64_RELOC_UNSIGNED, + ) + }; + for pair in relocations.as_chunks::<2>().0 { + assert_eq!(pair[0].0, pair[1].0); + for (relocation, r_type) in pair.iter().zip([subtractor, unsigned]) { + assert_eq!( + relocation.1.flags(), + object::RelocationFlags::MachO { + r_type, + r_pcrel: false, + r_length: 3, + } + ); + } + } + assert!(relocations[0].0 > relocations[2].0); + let compact = file.section_by_name("__compact_unwind").unwrap(); + assert_eq!(compact.data().unwrap().len(), 64); + for entry in compact.data().unwrap().as_chunks::<32>().0 { + assert_eq!( + u32::from_le_bytes(entry[12..16].try_into().unwrap()), + encoding + ); + } + } + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/main.rs b/experiments/gpu-rust-compiler/compiler/src/main.rs new file mode 100644 index 00000000000..d0c42e5de0f --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/main.rs @@ -0,0 +1,588 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[allow(dead_code)] // The analysis API is also used by the benchmark executables. +mod analysis; +#[allow(dead_code)] // Resident replay is an analysis benchmark operation. +mod cuda; +#[allow(dead_code)] // Calibration and indexed operations share this implementation. +mod executor; +mod frontend; +mod ir; +mod llvm; +mod metal; +#[allow(dead_code)] // Profiles are generated by the native benchmark executable. +mod routing; + +use std::{ + env, fs, + io::{self, BufRead, Write}, + path::{Path, PathBuf}, + process::{self, Command}, + sync::Arc, + time::Instant, +}; + +#[derive(Clone, Copy, Debug)] +enum Backend { + Cpu, + Metal, + Hybrid, + Cuda, + CudaHybrid, +} +impl Backend { + fn parse(value: &str) -> Result { + match value { + "cpu" => Ok(Self::Cpu), + "metal" => Ok(Self::Metal), + "hybrid" => Ok(Self::Hybrid), + "cuda" => Ok(Self::Cuda), + "cuda-hybrid" => Ok(Self::CudaHybrid), + _ => Err(format!( + "Unsupported native backend {value}; choose cpu, metal, hybrid, cuda, or cuda-hybrid" + )), + } + } + fn name(self) -> &'static str { + match self { + Self::Cpu => "cpu", + Self::Metal => "metal", + Self::Hybrid => "hybrid", + Self::Cuda => "cuda", + Self::CudaHybrid => "cuda-hybrid", + } + } +} +struct Options { + source: Option, + emit: Option, + output: Option, + report: Option, + backend: Option, + shader: PathBuf, + serve: bool, + verify: bool, + cuda_library: PathBuf, + cuda_algorithm: cuda::Algorithm, + route_profile: Option, + objective: String, + cpu_workers: usize, +} +fn options() -> Result, String> { + let mut result = Options { + source: None, + emit: None, + output: None, + report: None, + backend: None, + shader: Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("src/liveness.metal"), + serve: false, + verify: false, + cuda_library: env::var_os("GPU_LAB_CUDA_LIBRARY") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".build/cuda/libgpulab_cuda.so")), + cuda_algorithm: cuda::Algorithm::Dense, + route_profile: None, + objective: "latency".into(), + cpu_workers: 0, + }; + let mut arguments = env::args().skip(1); + while let Some(argument) = arguments.next() { + match argument.as_str() { + "--help" | "-h" => { + println!("Usage: gpu-rust-compiler SOURCE.rs --output EXECUTABLE --backend cpu|metal|hybrid|cuda|cuda-hybrid [--verify] [--emit-llvm FILE] [--report FILE]"); + println!(" gpu-rust-compiler --serve --backend cpu|metal|hybrid|cuda|cuda-hybrid [--verify] [--shader FILE]"); + println!("CUDA options: --cuda-library LIBRARY --cuda-algorithm dense|sparse --route-profile PROFILE --mode latency|throughput"); + println!("CPU options: --cpu-workers COUNT (0 uses available CPUs)"); + println!("Worker input: compileSOURCEOUTPUTREPORT (REPORT may be empty). One JSON response per request."); + println!("Raw frontend compatibility: SOURCE.rs --emit-llvm FILE (without --backend or --output)."); + return Ok(None); + } + "--serve" => result.serve = true, + "--verify" => result.verify = true, + "--cuda-library" => { + result.cuda_library = + PathBuf::from(arguments.next().ok_or("Missing CUDA library path")?) + } + "--cuda-algorithm" => { + result.cuda_algorithm = + match arguments.next().ok_or("Missing CUDA algorithm")?.as_str() { + "dense" => cuda::Algorithm::Dense, + "sparse" => cuda::Algorithm::Sparse, + value => return Err(format!("Unsupported CUDA algorithm {value}")), + } + } + "--route-profile" => { + result.route_profile = Some(PathBuf::from( + arguments.next().ok_or("Missing route profile path")?, + )) + } + "--cpu-workers" => { + result.cpu_workers = arguments + .next() + .ok_or("Missing CPU worker count")? + .parse::() + .map_err(|_| "Invalid CPU worker count")? + } + "--mode" => { + result.objective = arguments.next().ok_or("Missing execution mode")?; + if !matches!(result.objective.as_str(), "latency" | "throughput") { + return Err("Execution mode must be latency or throughput".into()); + } + } + "--emit-llvm" => { + result.emit = Some(PathBuf::from(arguments.next().ok_or("Missing LLVM path")?)) + } + "--output" => { + result.output = Some(PathBuf::from( + arguments.next().ok_or("Missing executable path")?, + )) + } + "--report" => { + result.report = Some(PathBuf::from( + arguments.next().ok_or("Missing report path")?, + )) + } + "--shader" => { + result.shader = PathBuf::from(arguments.next().ok_or("Missing shader path")?) + } + "--backend" => { + result.backend = Some(Backend::parse(&arguments.next().ok_or("Missing backend")?)?) + } + _ if !argument.starts_with('-') && result.source.is_none() => { + result.source = Some(PathBuf::from(argument)) + } + _ => return Err(format!("Unknown option {argument}")), + } + } + if !result.serve && result.source.is_none() { + return Err("A source path or --serve is required".into()); + } + if result.serve && result.source.is_some() { + return Err("--serve receives source paths through stdin".into()); + } + Ok(Some(result)) +} +fn ensure_parent(path: &Path) -> Result<(), String> { + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + Ok(()) +} +fn json_string(text: &str) -> String { + let mut result = String::from("\""); + for ch in text.chars() { + match ch { + '"' => result.push_str("\\\""), + '\\' => result.push_str("\\\\"), + '\n' => result.push_str("\\n"), + '\r' => result.push_str("\\r"), + '\t' => result.push_str("\\t"), + ch if ch < '\u{20}' => result.push_str(&format!("\\u{:04x}", ch as u32)), + ch => result.push(ch), + } + } + result.push('"'); + result +} +fn ms(start: Instant) -> f64 { + start.elapsed().as_secs_f64() * 1000.0 +} +struct Report { + backend: Backend, + source: PathBuf, + output: PathBuf, + request: usize, + total: f64, + frontend: f64, + packing: f64, + analysis: f64, + pruning: f64, + codegen: f64, + removed: usize, + functions: usize, + cpu_functions: usize, + gpu_functions: usize, + verified: bool, + pipeline_creations: usize, + gpu_submissions: usize, + gpu: Option, + cuda: Option, + cuda_capabilities: Option, + cpu_workers: usize, + cpu_requests: usize, + cpu_pool_reused: bool, + cpu_pool_submissions: usize, + cpu_initialization: f64, + routing_reason: String, + objective: String, + cuda_algorithm: cuda::Algorithm, +} +impl Report { + fn json(&self) -> String { + let (gpu_ms, reused, allocations, sweeps) = if let Some(gpu) = self.cuda { + ( + format!("{:.6}", gpu.gpu_ms), + gpu.reused_context != 0, + gpu.device_allocations, + gpu.max_sweeps, + ) + } else if let Some(gpu) = self.gpu { + ( + format!("{:.6}", gpu.gpu_ms), + gpu.reused_pipeline != 0, + gpu.buffer_allocations, + gpu.max_sweeps, + ) + } else { + ("null".into(), false, 0, 0) + }; + let mut result = format!(concat!( + "{{\"status\":\"ok\",\"compiler\":\"native scalar Rust-subset compiler\",", + "\"backend\":{},\"source\":{},\"output\":{},\"process_id\":{},\"request_number\":{},", + "\"total_compile_ms\":{:.6},\"frontend_and_structured_ir_ms\":{:.6},\"native_packing_ms\":{:.6},", + "\"analysis_ms\":{:.6},\"dead_code_pruning_ms\":{:.6},\"native_codegen_and_link_ms\":{:.6},", + "\"dead_scalar_instructions_removed\":{},\"functions\":{},\"cpu_functions\":{},\"actual_gpu_functions\":{},", + "\"verified_against_cpu\":{},\"gpu_pipeline_creations\":{},\"gpu_submissions_total\":{},", + "\"gpu_execution_ms\":{},\"gpu_pipeline_reused\":{},\"gpu_buffer_allocations\":{},\"gpu_max_sweeps\":{},", + "\"analysis_transport\":\"native memory; no textual IR parsing, JSON handoff, or analysis subprocess\",", + "\"remaining_codegen_subprocess\":\"clang\"}}"), + json_string(self.backend.name()), json_string(&self.source.to_string_lossy()), json_string(&self.output.to_string_lossy()), + process::id(), self.request, self.total, self.frontend, self.packing, self.analysis, self.pruning, self.codegen, + self.removed, self.functions, self.cpu_functions, self.gpu_functions, self.verified, + self.pipeline_creations, self.gpu_submissions, gpu_ms, reused, allocations, sweeps); + result.pop(); + result.push_str(&format!(concat!( + ",\"cpu_worker_threads\":{},\"cpu_worker_pool_creations\":1,\"cpu_requests_total\":{},", + "\"cpu_pool_reused\":{},\"cpu_pool_submissions\":{},\"cpu_initialization_ms\":{:.6},\"routing_reason\":{},", + "\"execution_objective\":{},\"cuda_algorithm\":{},\"cuda_stats\":{},\"cuda_capabilities\":{}}}"), + self.cpu_workers, self.cpu_requests, self.cpu_pool_reused, self.cpu_pool_submissions, self.cpu_initialization, + json_string(&self.routing_reason), json_string(&self.objective), + json_string(match self.cuda_algorithm { cuda::Algorithm::Dense => "dense", cuda::Algorithm::Sparse => "sparse" }), + self.cuda.map_or_else(|| "null".into(), |stats| stats.json()), + self.cuda_capabilities.as_ref().map_or_else(|| "null".into(), |capabilities| capabilities.json()))); + result + } +} +struct Compiler { + backend: Backend, + shader: PathBuf, + verify: bool, + metal: Option, + pipeline_creations: usize, + requests: usize, + cpu: executor::CpuExecutor, + cuda: Option, + cuda_library: PathBuf, + cuda_algorithm: cuda::Algorithm, + route_profile: Option, + objective: String, +} +impl Compiler { + fn new(options: &Options) -> Self { + Self { + backend: options.backend.unwrap_or(Backend::Hybrid), + shader: options.shader.clone(), + verify: options.verify, + metal: None, + pipeline_creations: 0, + requests: 0, + cpu: executor::CpuExecutor::new(options.cpu_workers), + cuda: None, + cuda_library: options.cuda_library.clone(), + cuda_algorithm: options.cuda_algorithm, + route_profile: options.route_profile.clone(), + objective: options.objective.clone(), + } + } + fn gpu(&mut self) -> Result<&mut metal::Context, String> { + if self.metal.is_none() { + self.metal = Some(metal::Context::new(&self.shader)?); + self.pipeline_creations += 1; + } + Ok(self.metal.as_mut().unwrap()) + } + fn cuda_gpu(&mut self) -> Result<&mut cuda::Context, String> { + if self.cuda.is_none() { + self.cuda = Some( + cuda::Context::new(&self.cuda_library) + .map_err(|error| format!("CUDA adapter unavailable: {error}"))?, + ); + self.pipeline_creations += 1; + } + Ok(self.cuda.as_mut().unwrap()) + } + fn compile( + &mut self, + source: &Path, + output: &Path, + emit: Option<&Path>, + report: Option<&Path>, + ) -> Result { + self.requests += 1; + let start = Instant::now(); + let source_text = + fs::read_to_string(source).map_err(|e| format!("{}: {e}", source.display()))?; + let program = frontend::parse(&source_text)?; + let module = llvm::emit_structured(&program)?; + let frontend = ms(start); + let stage = Instant::now(); + let packed = Arc::new(analysis::prepare(&module)?); + let packing = ms(stage); + let stage = Instant::now(); + let mut cpu_functions = module.functions.len(); + let mut gpu_functions = 0; + let mut gpu_stats = None; + let mut cuda_stats = None; + let mut selected_cuda_algorithm = self.cuda_algorithm; + let mut routing_reason = "Explicit backend selection".to_string(); + let solution = match self.backend { + Backend::Cpu => self.cpu.solve(Arc::clone(&packed)), + Backend::Cuda => { + let algorithm = self.cuda_algorithm; + self.cuda_gpu()?.submit(&packed, algorithm)?; + let (solution, stats) = self.cuda_gpu()?.finish()?; + cpu_functions = 0; + gpu_functions = module.functions.len(); + cuda_stats = Some(stats); + solution + } + Backend::CudaHybrid => { + if let Some(path) = self.route_profile.as_ref() { + let profile = routing::RouteProfile::load(path)?; + let objective = if self.objective == "throughput" { + routing::Objective::Throughput + } else { + routing::Objective::Latency + }; + // Check shape eligibility before paying for CUDA startup. + // Actual hardware identity is checked before any GPU work; + // this preliminary plan cannot authorize a submission. + if profile + .plan(&packed, &profile.hardware_identity, objective) + .gpu_indices + .is_empty() + { + routing_reason = + "No measured GPU win for this workload and objective; CPU selected" + .into(); + self.cpu.solve(Arc::clone(&packed)) + } else { + let workers = self.cpu.workers(); + let capabilities = self.cuda_gpu()?.capabilities(); + let identity = + routing::hardware_identity(&capabilities.fingerprint(workers), workers); + let plan = profile.plan(&packed, &identity, objective); + routing_reason = plan.reason; + if plan.gpu_indices.is_empty() { + if let Some(workers) = plan.cpu_worker_limit { + self.cpu.set_worker_limit(&packed, plan.cpu_mode, workers)?; + } + self.cpu.solve_mode(Arc::clone(&packed), plan.cpu_mode) + } else { + let selected = analysis::select_packed(&packed, &plan.gpu_indices)?; + let cpu_selection = + executor::AnalysisSelection::new(&packed, &plan.cpu_indices)?; + let algorithm = match plan.algorithm { + routing::GpuAlgorithm::Dense => cuda::Algorithm::Dense, + routing::GpuAlgorithm::Sparse => cuda::Algorithm::Sparse, + }; + selected_cuda_algorithm = algorithm; + self.cuda_gpu()?.submit(&selected, algorithm)?; + // Independent CPU functions execute after submission, + // while the CUDA stream owns its input snapshot. + let workers = plan + .cpu_worker_limit + .unwrap_or_else(|| self.cpu.worker_limit(&packed, plan.cpu_mode)); + let cpu_values = self.cpu.solve_selection_with_workers( + Arc::clone(&packed), + &cpu_selection, + plan.cpu_mode, + workers, + ); + let (gpu_values, stats) = self.cuda_gpu()?.finish()?; + let mut values = vec![0; packed.total_cells]; + cpu_values.scatter_into(&mut values)?; + analysis::scatter(&selected, &gpu_values, &packed, &mut values)?; + cpu_functions = plan.cpu_indices.len(); + gpu_functions = plan.gpu_indices.len(); + cuda_stats = Some(stats); + values + } + } + } else { + routing_reason = "No measured route profile; CPU selected".into(); + self.cpu.solve(Arc::clone(&packed)) + } + } + Backend::Metal => { + self.gpu()?.submit(&packed)?; + let (solution, stats) = self.gpu()?.finish()?; + cpu_functions = 0; + gpu_functions = module.functions.len(); + gpu_stats = Some(stats); + solution + } + Backend::Hybrid => { + let plan = analysis::hybrid_plan(&packed); + routing_reason = format!("Metal hybrid: {}", plan.reason); + if plan.gpu_indices.is_empty() { + self.cpu.solve(Arc::clone(&packed)) + } else { + let gpu_pack = analysis::prepare_subset(&module, &plan.gpu_indices)?; + let cpu_pack = Arc::new(analysis::prepare_subset(&module, &plan.cpu_indices)?); + self.gpu()?.submit(&gpu_pack)?; + let cpu_values = self.cpu.solve(Arc::clone(&cpu_pack)); + let (gpu_values, stats) = self.gpu()?.finish()?; + let mut result = vec![0; packed.total_cells]; + analysis::scatter(&cpu_pack, &cpu_values, &packed, &mut result)?; + analysis::scatter(&gpu_pack, &gpu_values, &packed, &mut result)?; + cpu_functions = plan.cpu_indices.len(); + gpu_functions = plan.gpu_indices.len(); + gpu_stats = Some(stats); + result + } + } + }; + if self.verify && solution != analysis::solve_cpu_serial(&packed) { + return Err("Native selected-backend liveness disagrees with CPU oracle".into()); + } + let analysis = ms(stage); + let stage = Instant::now(); + let (optimized, removed) = analysis::prune(&module, &packed, &solution)?; + let pruning = ms(stage); + let stage = Instant::now(); + let ir_path = emit + .map(Path::to_owned) + .unwrap_or_else(|| output.with_extension("native.ll")); + ensure_parent(&ir_path)?; + ensure_parent(output)?; + fs::write(&ir_path, optimized).map_err(|e| e.to_string())?; + let generated = Command::new("clang") + .args(["-Wno-override-module", "-O0"]) + .arg(&ir_path) + .arg("-o") + .arg(output) + .output() + .map_err(|e| format!("clang: {e}"))?; + if !generated.status.success() { + return Err(format!( + "Native code generation failed: {}", + String::from_utf8_lossy(&generated.stderr) + )); + } + let codegen = ms(stage); + let result = Report { + backend: self.backend, + source: source.to_owned(), + output: output.to_owned(), + request: self.requests, + total: ms(start), + frontend, + packing, + analysis, + pruning, + codegen, + removed, + functions: module.functions.len(), + cpu_functions, + gpu_functions, + verified: self.verify, + pipeline_creations: self.pipeline_creations, + gpu_submissions: self.metal.as_ref().map_or(0, |gpu| gpu.submissions) + + self.cuda.as_ref().map_or(0, |gpu| gpu.submissions), + gpu: gpu_stats, + cuda: cuda_stats, + cuda_capabilities: self.cuda.as_ref().map(|context| context.capabilities()), + cpu_workers: self.cpu.workers(), + cpu_requests: self.cpu.requests(), + cpu_pool_reused: self.cpu.pool_reused(), + cpu_pool_submissions: self.cpu.pool_submissions(), + cpu_initialization: self.cpu.initialization_ms(), + routing_reason, + objective: self.objective.clone(), + cuda_algorithm: selected_cuda_algorithm, + }; + if let Some(path) = report { + ensure_parent(path)?; + fs::write(path, result.json() + "\n").map_err(|e| e.to_string())?; + } + Ok(result) + } +} +fn run() -> Result<(), String> { + let Some(options) = options()? else { + return Ok(()); + }; + if options.serve { + let mut compiler = Compiler::new(&options); + let stdin = io::stdin(); + let mut stdout = io::stdout().lock(); + for line in stdin.lock().lines() { + let line = line.map_err(|e| e.to_string())?; + if line == "quit" { + break; + } + let fields: Vec<_> = line.split('\t').collect(); + let result = if fields.len() == 4 + && fields[0] == "compile" + && !fields[1].is_empty() + && !fields[2].is_empty() + { + compiler.compile( + Path::new(fields[1]), + Path::new(fields[2]), + None, + (!fields[3].is_empty()).then(|| Path::new(fields[3])), + ) + } else { + Err("Expected compileSOURCEOUTPUTREPORT; paths cannot contain tabs or newlines".into()) + }; + let response = match result { + Ok(report) => report.json(), + Err(error) => format!("{{\"status\":\"error\",\"error\":{}}}", json_string(&error)), + }; + writeln!(stdout, "{response}").map_err(|e| e.to_string())?; + stdout.flush().map_err(|e| e.to_string())?; + } + return Ok(()); + } + let source = options.source.as_ref().unwrap(); + if options.output.is_none() && options.backend.is_none() { + let emit = options + .emit + .as_ref() + .ok_or("--emit-llvm or --output is required")?; + let program = frontend::parse(&fs::read_to_string(source).map_err(|e| e.to_string())?)?; + ensure_parent(emit)?; + fs::write(emit, llvm::emit(&program)?).map_err(|e| e.to_string())?; + return Ok(()); + } + let output = options + .output + .as_ref() + .ok_or("Native compilation requires --output")?; + let mut compiler = Compiler::new(&options); + println!( + "{}", + compiler + .compile( + source, + output, + options.emit.as_deref(), + options.report.as_deref() + )? + .json() + ); + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("gpu-rust-compiler: {error}"); + process::exit(1); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/metal.rs b/experiments/gpu-rust-compiler/compiler/src/metal.rs new file mode 100644 index 00000000000..dafe2b6533d --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/metal.rs @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::analysis::{GPUGroup, PackedAnalysis}; +use std::path::Path; + +#[repr(C)] +#[derive(Default, Clone, Copy, Debug)] +pub struct Stats { + pub gpu_ms: f64, + pub total_ms: f64, + pub max_sweeps: u32, + pub reused_pipeline: u32, + pub buffer_allocations: u32, +} + +#[cfg(target_os = "macos")] +mod platform { + use super::*; + use std::{ + ffi::{c_char, c_int, c_void, CStr, CString}, + ptr::NonNull, + }; + + #[repr(C)] + struct Input { + function_count: u32, + block_count: u32, + cell_count: u32, + group_count: u32, + max_blocks: u32, + edge_count: u32, + groups: *const GPUGroup, + successor_offsets: *const u32, + successors: *const u32, + uses: *const u32, + defs: *const u32, + phi_out: *const u32, + } + #[link(name = "gpulab_metal")] + unsafe extern "C" { + fn gpulab_create(shader: *const c_char, error: *mut c_char, capacity: usize) + -> *mut c_void; + fn gpulab_submit( + context: *mut c_void, + input: *const Input, + error: *mut c_char, + capacity: usize, + ) -> c_int; + fn gpulab_finish( + context: *mut c_void, + output: *mut u32, + count: usize, + stats: *mut Stats, + error: *mut c_char, + capacity: usize, + ) -> c_int; + fn gpulab_destroy(context: *mut c_void); + } + + fn error_message(error: &[c_char]) -> String { + // The native bridge always NUL terminates a nonempty error buffer. + unsafe { CStr::from_ptr(error.as_ptr()) } + .to_string_lossy() + .into_owned() + } + + pub struct Context { + pointer: NonNull, + in_flight_cells: Option, + pub submissions: usize, + } + impl Context { + pub fn new(shader: &Path) -> Result { + let shader = CString::new(shader.as_os_str().as_encoded_bytes()) + .map_err(|_| "Shader path contains NUL")?; + let mut error = [0 as c_char; 1024]; + let pointer = + unsafe { gpulab_create(shader.as_ptr(), error.as_mut_ptr(), error.len()) }; + Ok(Self { + pointer: NonNull::new(pointer).ok_or_else(|| error_message(&error))?, + in_flight_cells: None, + submissions: 0, + }) + } + pub fn submit(&mut self, packed: &PackedAnalysis) -> Result<(), String> { + if self.in_flight_cells.is_some() { + return Err("Native GPU context already has an in-flight pass".into()); + } + // Public packed fields must not expose short slices to native reads. + if packed.row_offsets.len() < 2 + || packed.groups.is_empty() + || packed.functions.is_empty() + || packed.total_cells == 0 + || packed.successor_offsets.len() != packed.row_offsets.len() + || packed.uses.len() != packed.total_cells + || packed.defs.len() != packed.total_cells + || packed.phi_out.len() != packed.total_cells + { + return Err("Invalid native GPU slice dimensions".into()); + } + let count = |value: usize| { + u32::try_from(value).map_err(|_| "GPU input exceeds u32 dimensions".to_string()) + }; + let input = Input { + function_count: count(packed.functions.len())?, + block_count: count(packed.block_count())?, + cell_count: count(packed.total_cells)?, + group_count: count(packed.groups.len())?, + max_blocks: count(packed.max_blocks())?, + edge_count: count(packed.successors.len())?, + groups: packed.groups.as_ptr(), + successor_offsets: packed.successor_offsets.as_ptr(), + successors: packed.successors.as_ptr(), + uses: packed.uses.as_ptr(), + defs: packed.defs.as_ptr(), + phi_out: packed.phi_out.as_ptr(), + }; + let mut error = [0 as c_char; 1024]; + if unsafe { + gpulab_submit( + self.pointer.as_ptr(), + &input, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + return Err(error_message(&error)); + } + self.in_flight_cells = Some(packed.total_cells); + self.submissions += 1; + Ok(()) + } + pub fn finish(&mut self) -> Result<(Vec, Stats), String> { + let cells = self.in_flight_cells.take().ok_or("No in-flight GPU pass")?; + let mut result = vec![0; cells]; + let mut stats = Stats::default(); + let mut error = [0 as c_char; 1024]; + if unsafe { + gpulab_finish( + self.pointer.as_ptr(), + result.as_mut_ptr(), + result.len(), + &mut stats, + error.as_mut_ptr(), + error.len(), + ) + } != 0 + { + return Err(error_message(&error)); + } + Ok((result, stats)) + } + } + impl Drop for Context { + fn drop(&mut self) { + unsafe { gpulab_destroy(self.pointer.as_ptr()) }; + } + } +} + +#[cfg(not(target_os = "macos"))] +mod platform { + use super::*; + pub struct Context { + pub submissions: usize, + } + impl Context { + pub fn new(_: &Path) -> Result { + Err("Native Metal requires macOS; no GPU fallback was run".into()) + } + pub fn submit(&mut self, _: &PackedAnalysis) -> Result<(), String> { + Err("Metal unavailable".into()) + } + pub fn finish(&mut self) -> Result<(Vec, Stats), String> { + Err("Metal unavailable".into()) + } + } +} +pub use platform::Context; diff --git a/experiments/gpu-rust-compiler/compiler/src/native_codegen.rs b/experiments/gpu-rust-compiler/compiler/src/native_codegen.rs new file mode 100644 index 00000000000..2efa053afee --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/native_codegen.rs @@ -0,0 +1,348 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Physical GPU adapters for the schema-1 leaf emission fixture. Full Rust +//! lowering, object metadata and Cargo's backend interface are separate work. +use crate::{ + codegen_service::*, + machine_codegen::{self, MachineTarget, PackedFunction, PackedInstruction, Packet}, +}; +use std::{ + ffi::{c_char, c_int, c_void, CStr, CString}, + path::Path, + ptr::NonNull, + sync::{Arc, Mutex}, +}; + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug, serde::Serialize)] +pub struct NativeStats { + pub gpu_ms: f64, + pub total_ms: f64, + pub host_prefix_ms: f64, + pub count_gpu_ms: f64, + pub emit_gpu_ms: f64, + pub kernel_dispatches: u64, + pub output_bytes: u64, + pub buffer_allocations: u32, + pub reused_pipeline: u32, +} +#[repr(C)] +struct Input { + function_count: u32, + instruction_count: u32, + target: u32, + reserved: u32, + output_capacity: u64, + functions: *const PackedFunction, + instructions: *const PackedInstruction, +} +type Create = unsafe extern "C" fn(*const c_char, u32, *mut c_char, usize) -> *mut c_void; +type DeviceId = unsafe extern "C" fn(*mut c_void, *mut c_char, usize) -> c_int; +type Submit = unsafe extern "C" fn(*mut c_void, *const Input, *mut c_char, usize) -> c_int; +type Finish = unsafe extern "C" fn( + *mut c_void, + *mut u8, + usize, + *mut u32, + *mut u32, + usize, + *mut NativeStats, + *mut c_char, + usize, +) -> c_int; +type Destroy = unsafe extern "C" fn(*mut c_void); + +#[cfg_attr(target_os = "linux", link(name = "dl"))] +unsafe extern "C" { + fn dlopen(path: *const c_char, flags: c_int) -> *mut c_void; + fn dlsym(library: *mut c_void, name: *const c_char) -> *mut c_void; + fn dlclose(library: *mut c_void) -> c_int; + fn dlerror() -> *const c_char; +} +fn cerror(error: &[c_char]) -> String { + unsafe { CStr::from_ptr(error.as_ptr()) } + .to_string_lossy() + .into_owned() +} +fn dl_error() -> String { + let ptr = unsafe { dlerror() }; + if ptr.is_null() { + "native codegen library loading failed".into() + } else { + unsafe { CStr::from_ptr(ptr) } + .to_string_lossy() + .into_owned() + } +} +struct Library(NonNull); +impl Library { + fn open(path: &Path) -> Result { + let path = CString::new(path.as_os_str().as_encoded_bytes()) + .map_err(|_| "library path contains NUL")?; + NonNull::new(unsafe { dlopen(path.as_ptr(), 2) }) + .map(Self) + .ok_or_else(dl_error) + } + fn symbol(&self, name: &[u8]) -> Result<*mut c_void, String> { + NonNull::new(unsafe { dlsym(self.0.as_ptr(), name.as_ptr().cast()) }) + .map(NonNull::as_ptr) + .ok_or_else(dl_error) + } +} +impl Drop for Library { + fn drop(&mut self) { + unsafe { + dlclose(self.0.as_ptr()); + } + } +} +struct Context { + _library: Library, + pointer: Option>, + submit: Submit, + finish: Finish, + destroy: Destroy, + device_id: String, + pending: bool, +} +// The context is accessed under one mutex; CUDA binds its device on every call. +// The DLL stays loaded until after its context is fenced and destroyed. +unsafe impl Send for Context {} +impl Drop for Context { + fn drop(&mut self) { + if let Some(p) = self.pointer.take() { + unsafe { + (self.destroy)(p.as_ptr()); + } + } + } +} + +pub struct NativeExecutor { + backend: GpuBackend, + target: MachineTarget, + context: Arc>, + prepared: Option<(BatchKey, Packet)>, + stats: Arc>>, +} +impl NativeExecutor { + pub fn new( + backend: GpuBackend, + target: MachineTarget, + library: &Path, + shader: Option<&Path>, + device: u32, + ) -> Result { + let library = Library::open(library)?; + let create: Create = unsafe { std::mem::transmute(library.symbol(b"gpuemit_create\0")?) }; + let device_id: DeviceId = + unsafe { std::mem::transmute(library.symbol(b"gpuemit_device_id\0")?) }; + let submit: Submit = unsafe { std::mem::transmute(library.symbol(b"gpuemit_submit\0")?) }; + let finish: Finish = unsafe { std::mem::transmute(library.symbol(b"gpuemit_finish\0")?) }; + let destroy: Destroy = + unsafe { std::mem::transmute(library.symbol(b"gpuemit_destroy\0")?) }; + let shader = CString::new( + shader + .map(|s| s.as_os_str().as_encoded_bytes()) + .unwrap_or(b""), + ) + .map_err(|_| "shader path contains NUL")?; + let mut error = [0 as c_char; 1024]; + let pointer = NonNull::new(unsafe { + create(shader.as_ptr(), device, error.as_mut_ptr(), error.len()) + }) + .ok_or_else(|| cerror(&error))?; + let mut identity = [0 as c_char; 1024]; + if unsafe { device_id(pointer.as_ptr(), identity.as_mut_ptr(), identity.len()) } != 0 { + unsafe { + destroy(pointer.as_ptr()); + } + return Err("native bridge could not identify its physical device".into()); + } + let id = cerror(&identity); + if id.is_empty() { + unsafe { + destroy(pointer.as_ptr()); + } + return Err("empty physical device identity".into()); + } + Ok(Self { + backend, + target, + context: Arc::new(Mutex::new(Context { + _library: library, + pointer: Some(pointer), + submit, + finish, + destroy, + device_id: id, + pending: false, + })), + prepared: None, + stats: Arc::new(Mutex::new(Vec::new())), + }) + } + pub fn stats(&self) -> Arc>> { + self.stats.clone() + } +} +impl BatchExecutor for NativeExecutor { + fn capabilities(&self) -> DeviceCapabilities { + let mut caps = DeviceCapabilities::physical_gpu( + self.backend, + self.context.lock().unwrap().device_id.clone(), + ); + caps.max_batch_bytes = 128 * 1024 * 1024; + caps + } + fn prepare(&mut self, batch: &FlatBatch) -> Result<(), String> { + self.prepared = Some(( + batch.key().clone(), + machine_codegen::pack(batch, self.target)?, + )); + Ok(()) + } + fn submit(&mut self, batch: Arc) -> Result, String> { + let (key, packet) = self + .prepared + .take() + .ok_or("native batch was not prepared")?; + if key != *batch.key() { + return Err("native prepared identity mismatch".into()); + } + let input = Input { + function_count: u32::try_from(packet.functions.len()) + .map_err(|_| "function count overflow")?, + instruction_count: u32::try_from(packet.instructions.len()) + .map_err(|_| "instruction count overflow")?, + target: packet.target.number(), + reserved: 0, + output_capacity: packet.output_capacity as u64, + functions: packet.functions.as_ptr(), + instructions: packet.instructions.as_ptr(), + }; + let mut context = self.context.lock().unwrap(); + if context.pending { + return Err("native context has an outstanding batch".into()); + } + let pointer = context + .pointer + .ok_or("native context was closed by a failed pending request")?; + let mut error = [0 as c_char; 1024]; + if unsafe { (context.submit)(pointer.as_ptr(), &input, error.as_mut_ptr(), error.len()) } + != 0 + { + return Err(cerror(&error)); + } + context.pending = true; + drop(context); + Ok(Box::new(NativePending { + context: self.context.clone(), + batch, + packet, + backend: self.backend, + stats: self.stats.clone(), + finished: false, + })) + } +} +struct NativePending { + context: Arc>, + batch: Arc, + packet: Packet, + backend: GpuBackend, + stats: Arc>>, + finished: bool, +} +impl PendingExecution for NativePending { + fn wait(mut self: Box) -> Result { + let mut output = vec![0; self.packet.output_capacity]; + let mut offsets = vec![0u32; self.packet.functions.len()]; + let mut lengths = offsets.clone(); + let mut stats = NativeStats::default(); + let mut error = [0 as c_char; 1024]; + let mut context = self.context.lock().unwrap(); + let pointer = context.pointer.ok_or("native context is closed")?; + let status = unsafe { + (context.finish)( + pointer.as_ptr(), + output.as_mut_ptr(), + output.len(), + offsets.as_mut_ptr(), + lengths.as_mut_ptr(), + offsets.len(), + &mut stats, + error.as_mut_ptr(), + error.len(), + ) + }; + if status != 0 { + return Err(cerror(&error)); + } + context.pending = false; + self.finished = true; + let device_id = context.device_id.clone(); + drop(context); + if stats.kernel_dispatches == 0 + || stats.output_bytes == 0 + || stats.output_bytes as usize > output.len() + { + return Err("no valid native GPU emission was returned".into()); + } + let mut functions = Vec::with_capacity(offsets.len()); + let mut expected = 0usize; + for (i, (&offset, &len)) in offsets.iter().zip(&lengths).enumerate() { + let start = offset as usize; + let end = start + .checked_add(len as usize) + .ok_or("native result range overflow")?; + if start != expected || len == 0 || end > stats.output_bytes as usize { + return Err("native output ranges are incomplete or overlap".into()); + } + let unwind = + if self.batch.abis()[self.batch.functions()[i].abi as usize].requires_unwind { + machine_codegen::unwind_descriptor(self.packet.target, len as usize) + } else { + Vec::new() + }; + functions.push(FunctionEmission { + symbol: self.packet.functions[i].symbol, + code: output[start..end].to_vec(), + relocations: Vec::new(), + unwind, + }); + expected = end; + } + if expected != stats.output_bytes as usize { + return Err("unused native output bytes".into()); + } + self.stats.lock().unwrap().push(stats); + // The returned code comes from the native GPU output buffer. Object + // publication and independent CPU/native execution checks happen later. + Ok(Emission { + functions, + execution: ExecutionRecord::PhysicalGpu { + backend: self.backend, + device_id, + kernel_dispatches: stats.kernel_dispatches, + kernel_output_used: true, + }, + }) + } +} +impl Drop for NativePending { + fn drop(&mut self) { + if !self.finished { + let mut context = self.context.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(pointer) = context.pointer.take() { + unsafe { + (context.destroy)(pointer.as_ptr()); + } + } + context.pending = false; + } + // Retain batch and packet until after wait or the destroy fence. + let _ = &self.batch; + } +} diff --git a/experiments/gpu-rust-compiler/compiler/src/routing.rs b/experiments/gpu-rust-compiler/compiler/src/routing.rs new file mode 100644 index 00000000000..e98b9469876 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/src/routing.rs @@ -0,0 +1,816 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Measured placement policy. Profiles are specific to hardware and objective, +//! and GPU admission requires separate heldout samples of total recurring cost. +//! Shape classes are input properties, never device-name thresholds. + +use crate::{ + analysis::{FunctionDesc, PackedAnalysis}, + executor::CpuMode, +}; +use std::{collections::BTreeMap, fs, path::Path}; + +/// Bind evidence to CPU capacity, compiled experimental revision, and the +/// complete GPU/driver/runtime fingerprint supplied by the native adapter. +pub fn hardware_identity(gpu_fingerprint: &str, worker_count: usize) -> String { + let model = fs::read_to_string("/proc/cpuinfo") + .ok() + .and_then(|text| { + text.lines().find_map(|line| { + line.strip_prefix("model name").and_then(|value| { + value + .split_once(':') + .map(|(_, name)| name.trim().to_string()) + }) + }) + }) + .unwrap_or_else(|| format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH)); + let available = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + let revision = std::env::var("CANDIDATE_SHA").unwrap_or_else(|_| "unversioned-local".into()); + format!("gpu={gpu_fingerprint}|cpu={model}|available={available}|workers={worker_count}|revision={revision}") +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Objective { + Latency, + Throughput, +} +impl Objective { + pub fn name(self) -> &'static str { + match self { + Self::Latency => "latency", + Self::Throughput => "throughput", + } + } + fn parse(s: &str) -> Result { + match s { + "latency" => Ok(Self::Latency), + "throughput" => Ok(Self::Throughput), + _ => Err("Unknown routing objective".into()), + } + } +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GpuAlgorithm { + Dense, + Sparse, +} +impl GpuAlgorithm { + pub fn name(self) -> &'static str { + match self { + Self::Dense => "dense", + Self::Sparse => "sparse", + } + } + fn parse(s: &str) -> Result { + match s { + "dense" => Ok(Self::Dense), + "sparse" => Ok(Self::Sparse), + _ => Err("Unknown GPU algorithm".into()), + } + } +} + +/// Powers-of-two dimension classes plus GEN/phi density and dependence shape. +/// Include aggregate batch size so a fast large batch cannot admit tiny inputs. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct ShapeBucket { + pub functions: usize, + pub cells: usize, + pub blocks: usize, + pub words: usize, + pub density: usize, + pub edges: usize, + pub depth: usize, + pub cycles: usize, +} +impl ShapeBucket { + pub fn from_packed(p: &PackedAnalysis) -> Self { + Self::from_functions(p, p.functions.iter()) + } + fn from_functions<'a>( + p: &PackedAnalysis, + functions: impl Iterator, + ) -> Self { + let ( + mut count, + mut cells, + mut max_blocks, + mut max_words, + mut used, + mut edges, + mut depth, + mut cycles, + ) = (0, 0, 0, 0, 0usize, 0, 0, 0); + for f in functions { + count += 1; + cells += f.block_count * f.words; + max_blocks = max_blocks.max(f.block_count); + max_words = max_words.max(f.words); + let end = f.row_base + f.block_count * f.words; + used += p.uses[f.row_base..end] + .iter() + .zip(&p.phi_out[f.row_base..end]) + .map(|(u, phi)| (u | phi).count_ones() as usize) + .sum::(); + edges += (p.successor_offsets[f.block_base + f.block_count] + - p.successor_offsets[f.block_base]) as usize; + let (function_depth, has_cycle) = forward_depth(p, f); + depth = depth.max(function_depth); + cycles |= usize::from(has_cycle); + } + let density = if used == 0 { + 0 + } else if used.saturating_mul(100) <= cells.saturating_mul(32) { + 1 + } else if used.saturating_mul(10) <= cells.saturating_mul(32) { + 2 + } else { + 3 + }; + // Edge density is normalized by maximum function dimensions and count; + // other shape fields still distinguish varying-width/size batches. + Self { + functions: log_class(count), + cells: log_class(cells), + blocks: log_class(max_blocks), + words: log_class(max_words), + density, + edges: log_class(edges.div_ceil(count.max(1))), + depth: log_class(depth), + cycles, + } + } + fn fields(self) -> [usize; 8] { + [ + self.functions, + self.cells, + self.blocks, + self.words, + self.density, + self.edges, + self.depth, + self.cycles, + ] + } + fn from_fields(v: &[usize]) -> Self { + Self { + functions: v[0], + cells: v[1], + blocks: v[2], + words: v[3], + density: v[4], + edges: v[5], + depth: v[6], + cycles: v[7], + } + } + fn function_key(mut self) -> Self { + self.functions = 0; + self.cells = 0; + self + } +} +fn log_class(n: usize) -> usize { + if n == 0 { + 0 + } else { + usize::BITS as usize - n.leading_zeros() as usize + } +} + +// Forward-only dependence depth is cheap and deterministic. Backedges are +// separately encoded; no unmeasured SCC decomposition is hidden in routing. +fn forward_depth(p: &PackedAnalysis, f: &FunctionDesc) -> (usize, bool) { + let mut lengths = vec![1usize; f.block_count]; + let mut cycle = false; + for local in 0..f.block_count { + let block = f.block_base + local; + for &target in &p.successors + [p.successor_offsets[block] as usize..p.successor_offsets[block + 1] as usize] + { + let target = target as usize - f.block_base; + if target > local { + lengths[target] = lengths[target].max(lengths[local] + 1); + } else { + cycle = true; + } + } + } + (lengths.into_iter().max().unwrap_or(0), cycle) +} +fn long_forward_chain(p: &PackedAnalysis, f: &FunctionDesc) -> bool { + if f.block_count < 128 { + return false; + } + let (depth, cycle) = forward_depth(p, f); + !cycle + && depth == f.block_count + && (0..f.block_count).all(|local| { + let block = f.block_base + local; + p.successor_offsets[block + 1] - p.successor_offsets[block] <= 1 + }) +} + +#[derive(Clone, Debug)] +pub struct CostInterval { + pub p10_ms: f64, + pub median_ms: f64, + pub p90_ms: f64, + pub samples: usize, +} +impl CostInterval { + fn new(samples: &[f64]) -> Result { + if samples.len() < 5 || samples.iter().any(|v| !v.is_finite() || *v <= 0.0) { + return Err( + "Routing calibration needs at least five finite positive elapsed samples".into(), + ); + } + let mut sorted = samples.to_vec(); + sorted.sort_unstable_by(f64::total_cmp); + let q = |fraction: f64| { + let position = (sorted.len() - 1) as f64 * fraction; + let first = position.floor() as usize; + let last = position.ceil() as usize; + sorted[first] + (sorted[last] - sorted[first]) * (position - first as f64) + }; + Ok(Self { + p10_ms: q(0.1), + median_ms: q(0.5), + p90_ms: q(0.9), + samples: samples.len(), + }) + } + fn valid(&self) -> bool { + self.samples >= 5 + && self.p10_ms.is_finite() + && self.p10_ms > 0.0 + && self.median_ms >= self.p10_ms + && self.p90_ms.is_finite() + && self.p90_ms >= self.median_ms + } +} +#[derive(Clone, Debug)] +pub struct RouteObservation { + pub bucket: ShapeBucket, + pub context_bucket: ShapeBucket, + pub cpu_mode: CpuMode, + pub cpu_worker_limit: usize, + pub algorithm: GpuAlgorithm, + pub training_cpu: CostInterval, + pub training_gpu: CostInterval, + pub heldout_cpu: CostInterval, + pub heldout_gpu: CostInterval, +} +impl RouteObservation { + fn stable_win(&self) -> bool { + // A 10% separation of the central 80% distributions rejects noise and + // removes the temptation to route from kernel-only speedups. + self.training_gpu.p90_ms < self.training_cpu.p10_ms * 0.9 + && self.heldout_gpu.p90_ms < self.heldout_cpu.p10_ms * 0.9 + } +} + +#[derive(Clone, Debug)] +pub struct RoutePlan { + pub cpu_indices: Vec, + pub gpu_indices: Vec, + pub cpu_mode: CpuMode, + pub cpu_worker_limit: Option, + pub algorithm: GpuAlgorithm, + pub reason: String, + pub cpu_ms: Option, + pub gpu_ms: Option, +} +#[derive(Clone, Debug)] +pub struct RouteProfile { + pub hardware_identity: String, + pub objective: Objective, + pub observations: Vec, +} +impl RouteProfile { + pub fn new(hardware_identity: String, objective: Objective) -> Self { + Self { + hardware_identity, + objective, + observations: Vec::new(), + } + } + /// GPU samples must be total recurring elapsed cost: subset packing and + /// staging, upload, kernel and convergence wait, download and assembly. + /// For mixed execution include the CPU remainder and overlap boundary. + #[allow(clippy::too_many_arguments)] + pub fn record( + &mut self, + p: &PackedAnalysis, + cpu_mode: CpuMode, + algorithm: GpuAlgorithm, + training_cpu_ms: &[f64], + training_gpu_total_ms: &[f64], + heldout_cpu_ms: &[f64], + heldout_gpu_total_ms: &[f64], + ) -> Result<(), String> { + self.record_with_context( + p, + p, + cpu_mode, + algorithm, + training_cpu_ms, + training_gpu_total_ms, + heldout_cpu_ms, + heldout_gpu_total_ms, + ) + } + + /// For heterogeneous placement, the CPU baseline covers the entire parent + /// and GPU elapsed covers candidate packing + GPU + CPU remainder + scatter. + /// Binding the parent shape prevents recycling a cheap remainder's evidence + /// for another program with an expensive CPU critical path. + #[allow(clippy::too_many_arguments)] + pub fn record_with_context( + &mut self, + p: &PackedAnalysis, + parent: &PackedAnalysis, + cpu_mode: CpuMode, + algorithm: GpuAlgorithm, + training_cpu_ms: &[f64], + training_gpu_total_ms: &[f64], + heldout_cpu_ms: &[f64], + heldout_gpu_total_ms: &[f64], + ) -> Result<(), String> { + self.record_with_context_and_workers( + p, + parent, + cpu_mode, + 0, + algorithm, + training_cpu_ms, + training_gpu_total_ms, + heldout_cpu_ms, + heldout_gpu_total_ms, + ) + } + + #[allow(clippy::too_many_arguments)] + pub fn record_with_context_and_workers( + &mut self, + p: &PackedAnalysis, + parent: &PackedAnalysis, + cpu_mode: CpuMode, + cpu_worker_limit: usize, + algorithm: GpuAlgorithm, + training_cpu_ms: &[f64], + training_gpu_total_ms: &[f64], + heldout_cpu_ms: &[f64], + heldout_gpu_total_ms: &[f64], + ) -> Result<(), String> { + let row = RouteObservation { + bucket: ShapeBucket::from_packed(p), + context_bucket: ShapeBucket::from_packed(parent), + cpu_mode, + cpu_worker_limit, + algorithm, + training_cpu: CostInterval::new(training_cpu_ms)?, + training_gpu: CostInterval::new(training_gpu_total_ms)?, + heldout_cpu: CostInterval::new(heldout_cpu_ms)?, + heldout_gpu: CostInterval::new(heldout_gpu_total_ms)?, + }; + self.observations.retain(|old| { + old.bucket != row.bucket + || old.context_bucket != row.context_bucket + || old.algorithm != algorithm + }); + self.observations.push(row); + Ok(()) + } + pub fn plan( + &self, + p: &PackedAnalysis, + hardware_identity: &str, + objective: Objective, + ) -> RoutePlan { + let mut plan = RoutePlan { + cpu_indices: p.functions.iter().map(|f| f.module_index).collect(), + gpu_indices: Vec::new(), + cpu_mode: CpuMode::Serial, + cpu_worker_limit: None, + algorithm: GpuAlgorithm::Dense, + reason: String::new(), + cpu_ms: None, + gpu_ms: None, + }; + if self.hardware_identity != hardware_identity { + plan.reason = "CPU default: calibration hardware identity differs".into(); + return plan; + } + if self.objective != objective { + plan.reason = "CPU default: calibration objective differs".into(); + return plan; + } + let complete_bucket = ShapeBucket::from_packed(p); + if let Some(best_cpu) = self + .observations + .iter() + .filter(|r| r.context_bucket == complete_bucket) + .min_by(|a, b| a.heldout_cpu.median_ms.total_cmp(&b.heldout_cpu.median_ms)) + { + plan.cpu_mode = best_cpu.cpu_mode; + plan.cpu_worker_limit = + (best_cpu.cpu_worker_limit > 0).then_some(best_cpu.cpu_worker_limit); + plan.cpu_ms = Some(best_cpu.heldout_cpu.median_ms); + } + let mut groups: BTreeMap> = BTreeMap::new(); + let mut chain_count = 0; + for f in &p.functions { + if long_forward_chain(p, f) { + chain_count += 1; + continue; + } + let key = ShapeBucket::from_functions(p, std::iter::once(f)).function_key(); + groups.entry(key).or_default().push(f); + } + let mut candidates = Vec::new(); + for functions in groups.values() { + let bucket = ShapeBucket::from_functions(p, functions.iter().copied()); + if let Some(row) = self + .observations + .iter() + .filter(|r| { + r.bucket == bucket && r.context_bucket == complete_bucket && r.stable_win() + }) + .min_by(|a, b| a.heldout_gpu.p90_ms.total_cmp(&b.heldout_gpu.p90_ms)) + { + candidates.push((row, functions)); + } + } + // One GPU algorithm per submission. Keep groups calibrated for other + // algorithms on CPU instead of inventing an unmeasured mixed cost. + if let Some((winner, functions)) = candidates.iter().max_by(|(a, _), (b, _)| { + (a.heldout_cpu.median_ms - a.heldout_gpu.median_ms) + .total_cmp(&(b.heldout_cpu.median_ms - b.heldout_gpu.median_ms)) + }) { + plan.algorithm = winner.algorithm; + plan.cpu_mode = winner.cpu_mode; + plan.cpu_worker_limit = + (winner.cpu_worker_limit > 0).then_some(winner.cpu_worker_limit); + // Admit one calibrated group. Combining independently measured + // groups changes the unmeasured packing and overlap boundary. + plan.gpu_indices + .extend(functions.iter().map(|f| f.module_index)); + plan.cpu_ms = Some(winner.heldout_cpu.median_ms); + plan.gpu_ms = Some(winner.heldout_gpu.median_ms); + plan.cpu_indices + .retain(|index| !plan.gpu_indices.contains(index)); + plan.reason=format!("{} GPU functions earned a >=10% separated training/heldout total elapsed win; {} structural long chains stay on CPU",plan.gpu_indices.len(),chain_count); + } else { + plan.reason=format!("CPU default: no calibrated shape bucket has a stable total elapsed GPU win; {chain_count} structural long chains stay on CPU"); + } + plan + } + pub fn save(&self, path: &Path) -> Result<(), String> { + fs::write(path, self.to_csv()).map_err(|e| format!("Cannot write route profile: {e}")) + } + pub fn load(path: &Path) -> Result { + let data = + fs::read_to_string(path).map_err(|e| format!("Cannot read route profile: {e}"))?; + Self::from_csv(&data) + } + pub fn to_csv(&self) -> String { + let identity = self + .hardware_identity + .as_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + let mut output = format!("GLC_ROUTE3,{identity},{}\n", self.objective.name()); + for row in &self.observations { + let fields = row + .bucket + .fields() + .iter() + .map(usize::to_string) + .collect::>() + .join(","); + let context_fields = row + .context_bucket + .fields() + .iter() + .map(usize::to_string) + .collect::>() + .join(","); + output.push_str(&format!( + "{fields},{context_fields},{},{},{},", + row.cpu_mode.name(), + row.algorithm.name(), + row.cpu_worker_limit + )); + let costs = [ + &row.training_cpu, + &row.training_gpu, + &row.heldout_cpu, + &row.heldout_gpu, + ]; + output.push_str( + &costs + .iter() + .map(|c| format!("{},{},{},{}", c.p10_ms, c.median_ms, c.p90_ms, c.samples)) + .collect::>() + .join(","), + ); + output.push('\n'); + } + output + } + pub fn from_csv(data: &str) -> Result { + if data.len() > 1_000_000 { + return Err("Route profile exceeds size limit".into()); + } + let mut lines = data.lines(); + let header = lines + .next() + .ok_or("Empty route profile")? + .split(',') + .collect::>(); + if header.len() != 3 + || header[0] != "GLC_ROUTE3" + || header[1].len() % 2 != 0 + || !header[1].bytes().all(|b| b.is_ascii_hexdigit()) + { + return Err("Invalid route profile header".into()); + } + let bytes = (0..header[1].len()) + .step_by(2) + .map(|index| { + u8::from_str_radix(&header[1][index..index + 2], 16) + .map_err(|_| "Invalid routing hardware identity".to_string()) + }) + .collect::, _>>()?; + let identity = + String::from_utf8(bytes).map_err(|_| "Invalid routing hardware identity UTF-8")?; + let mut profile = Self::new(identity, Objective::parse(header[2])?); + for line in lines { + let raw = line.split(',').collect::>(); + if raw.len() != 35 { + return Err("Invalid route profile row field count".into()); + } + let shape = raw[..16] + .iter() + .map(|s| { + s.parse::() + .map_err(|_| "Invalid route shape".to_string()) + }) + .collect::, _>>()?; + if shape.iter().any(|n| *n > usize::BITS as usize) + || shape[4] > 3 + || shape[7] > 1 + || shape[12] > 3 + || shape[15] > 1 + { + return Err("Route shape outside supported classes".into()); + } + let cpu_mode = match raw[16] { + "serial" => CpuMode::Serial, + "function_pool" => CpuMode::Functions, + "word_pool" => CpuMode::Words, + _ => return Err("Unknown CPU route mode".into()), + }; + let interval = |first: usize| -> Result { + let number = |index: usize| { + raw[index] + .parse::() + .map_err(|_| "Invalid routing elapsed cost".to_string()) + }; + let c = CostInterval { + p10_ms: number(first)?, + median_ms: number(first + 1)?, + p90_ms: number(first + 2)?, + samples: raw[first + 3] + .parse() + .map_err(|_| "Invalid routing sample count")?, + }; + if !c.valid() { + return Err("Invalid routing elapsed interval".into()); + } + Ok(c) + }; + profile.observations.push(RouteObservation { + bucket: ShapeBucket::from_fields(&shape), + context_bucket: ShapeBucket::from_fields(&shape[8..]), + cpu_mode, + cpu_worker_limit: raw[18] + .parse::() + .map_err(|_| "Invalid CPU worker limit")?, + algorithm: GpuAlgorithm::parse(raw[17])?, + training_cpu: interval(19)?, + training_gpu: interval(23)?, + heldout_cpu: interval(27)?, + heldout_gpu: interval(31)?, + }); + } + Ok(profile) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::prepare; + use crate::ir::{BlockIR, FunctionIR, InstructionIR, ModuleIR}; + fn pack(count: usize, blocks: usize) -> PackedAnalysis { + prepare(&ModuleIR { + text: String::new(), + preamble: String::new(), + postamble: String::new(), + functions: (0..count) + .map(|index| FunctionIR { + name: format!("f{index}"), + header: String::new(), + footer: String::new(), + value_count: 1, + phi_edges: Vec::new(), + blocks: (0..blocks) + .map(|b| BlockIR { + name: format!("b{b}"), + successors: if b + 1 < blocks { + vec![b + 1] + } else { + Vec::new() + }, + instructions: vec![InstructionIR { + text: String::new(), + definition: None, + uses: vec![0], + pure: false, + is_phi: false, + }], + }) + .collect(), + }) + .collect(), + }) + .unwrap() + } + #[test] + fn routing_defaults_to_cpu_for_missing_unstable_and_wrong_hardware_evidence() { + let packed = pack(32, 2); + let mut profile = RouteProfile::new("test-GPU|driver|CPU".into(), Objective::Latency); + assert!(profile + .plan(&packed, "test-GPU|driver|CPU", Objective::Latency) + .gpu_indices + .is_empty()); + profile + .record( + &packed, + CpuMode::Functions, + GpuAlgorithm::Dense, + &[1.0; 5], + &[0.95; 5], + &[1.0; 5], + &[0.95; 5], + ) + .unwrap(); + assert!(profile + .plan(&packed, "test-GPU|driver|CPU", Objective::Latency) + .gpu_indices + .is_empty()); + profile + .record( + &packed, + CpuMode::Functions, + GpuAlgorithm::Dense, + &[1.0; 5], + &[0.5; 5], + &[1.0; 5], + &[0.5; 5], + ) + .unwrap(); + assert_eq!( + profile + .plan(&packed, "test-GPU|driver|CPU", Objective::Latency) + .gpu_indices + .len(), + 32 + ); + assert!(profile + .plan(&packed, "another-GPU", Objective::Latency) + .gpu_indices + .is_empty()); + assert!(profile + .plan(&packed, "test-GPU|driver|CPU", Objective::Throughput) + .gpu_indices + .is_empty()); + assert!(profile + .plan(&pack(1, 2), "test-GPU|driver|CPU", Objective::Latency) + .gpu_indices + .is_empty()); + } + #[test] + fn heldout_loss_prevents_training_only_win_and_profile_roundtrip_is_strict() { + let packed = pack(16, 2); + let mut p = RouteProfile::new("gpu,uuid\nCPU".into(), Objective::Throughput); + p.record( + &packed, + CpuMode::Words, + GpuAlgorithm::Sparse, + &[1.0; 5], + &[0.5; 5], + &[1.0; 5], + &[1.1; 5], + ) + .unwrap(); + assert!(p + .plan(&packed, &p.hardware_identity, Objective::Throughput) + .gpu_indices + .is_empty()); + let copy = RouteProfile::from_csv(&p.to_csv()).unwrap(); + assert_eq!(copy.hardware_identity, p.hardware_identity); + assert_eq!(copy.observations.len(), 1); + assert!(RouteProfile::from_csv(&p.to_csv().replace("0.5", "NaN")).is_err()); + assert!(RouteProfile::from_csv("GLC_ROUTE3,zz,latency\n").is_err()); + assert!(RouteProfile::from_csv("GLC_ROUTE3,aé0,latency\n").is_err()); + assert!(p + .record( + &packed, + CpuMode::Serial, + GpuAlgorithm::Dense, + &[1.0; 4], + &[0.5; 5], + &[1.0; 5], + &[0.5; 5] + ) + .is_err()); + } + #[test] + fn earned_compact_batch_can_overlap_long_cpu_chains() { + let compact = pack(32, 2); + let chain = pack(1, 257); + let mut m = ModuleIR { + text: String::new(), + preamble: String::new(), + postamble: String::new(), + functions: Vec::new(), + }; + // Use the frontend fixture builder rather than stitching invalid CSR. + for index in 0..33 { + let blocks = if index == 32 { 257 } else { 2 }; + m.functions.push(FunctionIR { + name: format!("f{index}"), + header: String::new(), + footer: String::new(), + value_count: 1, + phi_edges: Vec::new(), + blocks: (0..blocks) + .map(|b| BlockIR { + name: format!("b{b}"), + successors: if b + 1 < blocks { + vec![b + 1] + } else { + Vec::new() + }, + instructions: vec![InstructionIR { + text: String::new(), + definition: None, + uses: vec![0], + pure: false, + is_phi: false, + }], + }) + .collect(), + }); + } + let mixed = prepare(&m).unwrap(); + let mut p = RouteProfile::new("same-device".into(), Objective::Latency); + p.record_with_context( + &compact, + &mixed, + CpuMode::Functions, + GpuAlgorithm::Dense, + &[1.0; 5], + &[0.5; 5], + &[1.0; 5], + &[0.5; 5], + ) + .unwrap(); + p.record( + &chain, + CpuMode::Functions, + GpuAlgorithm::Dense, + &[10.0; 5], + &[0.1; 5], + &[10.0; 5], + &[0.1; 5], + ) + .unwrap(); + let plan = p.plan(&mixed, "same-device", Objective::Latency); + assert_eq!(plan.gpu_indices, (0..32).collect::>()); + assert_eq!(plan.cpu_indices, vec![32]); + let other_parent = pack(33, 2); + assert!(p + .plan(&other_parent, "same-device", Objective::Latency) + .gpu_indices + .is_empty()); + } +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/bitcode_linker_contract.rs b/experiments/gpu-rust-compiler/compiler/tests/bitcode_linker_contract.rs new file mode 100644 index 00000000000..1c48440e8d1 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/bitcode_linker_contract.rs @@ -0,0 +1,554 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[path = "../src/bitcode_linker.rs"] +#[allow(dead_code)] +mod bitcode_linker; + +use bitcode_linker::{classify, InvocationKind}; +use bitcode_linker::{run_with, GpuOptions, ToolCall, ToolOutput, ToolRunner, WrapperOptions}; +use std::ffi::OsString; +use std::{ + fs, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +fn args(values: &[&str]) -> Vec { + values.iter().map(OsString::from).collect() +} + +#[test] +fn cargo_link_invocations_use_the_two_phase_bridge() { + let plan = classify(&args(&[ + "src/main.rs", + "--crate-name", + "app", + "--crate-type", + "bin", + "--emit=dep-info,metadata,link", + "--out-dir", + "/tmp/cargo-deps", + "-C", + "extra-filename=-123", + ])) + .unwrap(); + assert!(matches!(plan, InvocationKind::Compile(_))); +} + +#[test] +fn actual_crate_optimization_controls_llvm_codegen_including_size_and_package_overrides() { + for (flags, expected) in [ + (vec!["-C", "opt-level=1"], 1), + (vec!["-Copt-level=2"], 2), + (vec!["-Copt-level=3"], 3), + (vec!["-Copt-level=s"], 2), + (vec!["-Copt-level=z"], 2), + (vec!["-O"], 2), + (vec!["-Copt-level=1", "-C", "opt-level=2"], 2), + ] { + let mut values = vec!["src/main.rs", "--out-dir", "/tmp/llvm-level"]; + values.extend(flags); + let InvocationKind::Compile(plan) = classify(&args(&values)).unwrap() else { + panic!("compile was not selected"); + }; + assert_eq!(plan.llvm_codegen_opt_level, expected); + } +} + +#[test] +fn metadata_only_and_version_queries_pass_through_without_claiming_codegen() { + assert!(matches!( + classify(&args(&["--version"])).unwrap(), + InvocationKind::PassThrough(_) + )); + assert!(matches!( + classify(&args(&[ + "lib.rs", + "--emit=dep-info,metadata", + "--out-dir", + "/tmp/cargo-deps" + ])) + .unwrap(), + InvocationKind::PassThrough(_) + )); +} + +#[test] +fn response_files_and_explicit_emit_paths_are_left_to_original_rustc() { + assert!(matches!( + classify(&args(&["@rustc.args"])).unwrap(), + InvocationKind::PassThrough(_) + )); + assert!(matches!( + classify(&args(&[ + "lib.rs", + "--emit=link=/tmp/app", + "--out-dir", + "/tmp/deps" + ])) + .unwrap(), + InvocationKind::PassThrough(_) + )); +} + +struct Temporary(PathBuf); +static TEMP_NEXT: AtomicU64 = AtomicU64::new(0); +impl Temporary { + fn new() -> Self { + loop { + let number = TEMP_NEXT.fetch_add(1, Ordering::Relaxed); + let path = std::env::temp_dir().join(format!( + "object-bridge-test-{}-{number}", + std::process::id() + )); + match fs::create_dir(&path) { + Ok(()) => return Self(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("test staging: {error}"), + } + } + } +} +impl Drop for Temporary { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +struct FixtureRunner { + calls: Vec, + native_first: bool, + frontend_exit: i32, + controller_exit: i32, + link_exit: i32, +} +impl FixtureRunner { + fn new() -> Self { + Self { + calls: vec![], + native_first: false, + frontend_exit: 0, + controller_exit: 0, + link_exit: 0, + } + } +} +fn after(call: &ToolCall, key: &str) -> PathBuf { + let index = call.arguments.iter().position(|v| v == key).unwrap(); + PathBuf::from(&call.arguments[index + 1]) +} +fn native_object() -> Vec { + let mut object = object::write::Object::new( + object::BinaryFormat::Elf, + object::Architecture::X86_64, + object::Endianness::Little, + ); + let text = object.section_id(object::write::StandardSection::Text); + object.append_section_data(text, &[0xc3], 1); + object.write().unwrap() +} +impl ToolRunner for FixtureRunner { + fn run(&mut self, call: &ToolCall) -> Result { + self.calls.push(call.clone()); + let success = |code| ToolOutput { + exit_code: code, + stdout: vec![], + stderr: vec![], + }; + if call.arguments == [OsString::from("-vV")] { + return Ok(ToolOutput { + exit_code: 0, + stdout: b"rustc 1.98.1\nrelease: 1.98.1\n".to_vec(), + stderr: vec![], + }); + } + if call.arguments.first().is_some_and(|v| v == "--export-leaf") { + return Ok(ToolOutput { + exit_code: 9, + stdout: vec![], + stderr: b"fixture module contains unsupported globals".to_vec(), + }); + } + if call.arguments.iter().any(|v| v == "-Zno-link") { + if self.frontend_exit != 0 { + return Ok(success(self.frontend_exit)); + } + let stage = after(call, "--out-dir"); + fs::write(stage.join("app.rlink"), b"saved-metadata").unwrap(); + fs::write( + stage.join("app.app-hash-cgu.0.rcgu.o"), + if self.native_first { + native_object() + } else { + b"BC\xc0\xde-fixture".to_vec() + }, + ) + .unwrap(); + fs::write(stage.join("libapp.rmeta"), b"fixture-metadata").unwrap(); + fs::write( + stage.join("app.d"), + format!("{}/app: source.rs\n", stage.display()), + ) + .unwrap(); + return Ok(success(0)); + } + if call.arguments.iter().any(|v| v == "-Zlink-only") { + if self.link_exit == 0 { + fs::write(after(call, "--out-dir").join("app"), b"linked-fixture").unwrap(); + } + return Ok(success(self.link_exit)); + } + if call.program == Path::new("controller") { + if self.controller_exit != 0 { + return Ok(success(self.controller_exit)); + } + let bytes = native_object(); + fs::write(after(call, "--output"), &bytes).unwrap(); + let receipt = serde_json::json!({"backend":"llvm","target":"x86_64-unknown-linux-gnu","gpu_accelerated":false,"object_bytes":bytes.len(),"fallback_reason":null}); + fs::write( + after(call, "--report"), + serde_json::to_vec(&receipt).unwrap(), + ) + .unwrap(); + } + Ok(success(0)) + } +} +fn setup(temporary: &Temporary) -> (WrapperOptions, Vec) { + let out = temporary.0.join("deps"); + fs::create_dir(&out).unwrap(); + let source = temporary.0.join("source.rs"); + fs::write(&source, "fn main() {}\n").unwrap(); + let options = WrapperOptions { + controller: "controller".into(), + bridge: "bridge".into(), + llc: "llc".into(), + backend: "llvm".into(), + allow_fallback: false, + receipt_directory: temporary.0.join("receipts"), + quality: "fast".into(), + gpu: None, + capture_directory: None, + }; + let args = vec![ + source.into_os_string(), + "--emit=dep-info,metadata,link".into(), + "--out-dir".into(), + out.into_os_string(), + ]; + (options, args) +} + +#[test] +fn compiler_objects_are_replaced_once_before_rust_only_links_saved_metadata() { + let temporary = Temporary::new(); + let (options, args) = setup(&temporary); + let mut runner = FixtureRunner::new(); + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 0); + assert_eq!(outcome.report.status, "success"); + assert_eq!(outcome.report.units.len(), 1); + assert_eq!(outcome.report.units[0].backend, "llvm"); + assert!(!outcome.report.gpu_executed); + let frontend = runner + .calls + .iter() + .find(|c| c.arguments.iter().any(|v| v == "-Zno-link")) + .unwrap(); + assert!(frontend.bootstrap); + assert!(frontend + .arguments + .contains(&OsString::from("-Clinker-plugin-lto=yes"))); + let link = runner + .calls + .iter() + .find(|c| c.arguments.iter().any(|v| v == "-Zlink-only")) + .unwrap(); + assert!(link.bootstrap); + assert!(link.arguments[0].to_string_lossy().ends_with(".rlink")); + let out = temporary.0.join("deps"); + assert_eq!(fs::read(out.join("app")).unwrap(), b"linked-fixture"); + assert_eq!( + fs::read(out.join("libapp.rmeta")).unwrap(), + b"fixture-metadata" + ); + assert!(!fs::read_to_string(out.join("app.d")) + .unwrap() + .contains("gpu-object-bridge")); + assert!(!outcome.report.staging_directory.unwrap().exists()); +} + +#[test] +fn audit_capture_preserves_exact_frontend_bitcode_after_native_object_replacement() { + let temporary = Temporary::new(); + let (mut options, mut arguments) = setup(&temporary); + options.capture_directory = Some(temporary.0.join("capture")); + arguments.extend(args(&["-Copt-level=1"])); + let mut runner = FixtureRunner::new(); + let outcome = run_with(Path::new("rustc"), &arguments, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 0); + let path = outcome + .report + .capture_manifest + .expect("bitcode capture manifest was not retained"); + let manifest: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(manifest["schema"], "gpu-rust-bitcode-capture"); + assert_eq!(manifest["status"], "success"); + assert_eq!(manifest["llvm_codegen_opt_level"], 1); + assert_eq!(manifest["units"].as_array().unwrap().len(), 1); + let unit = &manifest["units"][0]; + let input = path + .parent() + .unwrap() + .join(unit["captured_filename"].as_str().unwrap()); + assert_eq!(fs::read(input).unwrap(), b"BC\xc0\xde-fixture"); + assert_eq!(unit["input_bytes"], 12); + assert_eq!(unit["input_sha256"], outcome.report.units[0].input_sha256); + assert_eq!(unit["target"], "x86_64-unknown-linux-gnu"); + assert_eq!(unit["emission_receipt"]["backend"], "llvm"); + assert!(!outcome.report.staging_directory.unwrap().exists()); +} + +#[test] +fn failed_emission_retains_audit_bitcode_without_a_successful_emission_claim() { + let temporary = Temporary::new(); + let (mut options, arguments) = setup(&temporary); + options.capture_directory = Some(temporary.0.join("capture")); + let mut runner = FixtureRunner::new(); + runner.controller_exit = 3; + let outcome = run_with(Path::new("rustc"), &arguments, &options, &mut runner).unwrap(); + assert_ne!(outcome.exit_code, 0); + let path = outcome.report.capture_manifest.unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(manifest["status"], "failed"); + assert!(manifest["units"][0]["emission_receipt"].is_null()); + let input = path + .parent() + .unwrap() + .join(manifest["units"][0]["captured_filename"].as_str().unwrap()); + assert_eq!(fs::read(input).unwrap(), b"BC\xc0\xde-fixture"); + assert!(!temporary.0.join("deps/app").exists()); +} + +#[test] +fn repeated_audit_captures_use_distinct_owned_directories_and_preserve_other_data() { + let temporary = Temporary::new(); + let (mut options, arguments) = setup(&temporary); + let root = temporary.0.join("capture"); + fs::create_dir(&root).unwrap(); + fs::write(root.join("other.bc"), b"other owned data").unwrap(); + options.capture_directory = Some(root.clone()); + let first = run_with( + Path::new("rustc"), + &arguments, + &options, + &mut FixtureRunner::new(), + ) + .unwrap(); + let second = run_with( + Path::new("rustc"), + &arguments, + &options, + &mut FixtureRunner::new(), + ) + .unwrap(); + assert_ne!( + first.report.capture_manifest, + second.report.capture_manifest + ); + assert_eq!( + fs::read(root.join("other.bc")).unwrap(), + b"other owned data" + ); + for path in [ + first.report.capture_manifest.unwrap(), + second.report.capture_manifest.unwrap(), + ] { + let value: serde_json::Value = serde_json::from_slice(&fs::read(path).unwrap()).unwrap(); + assert_eq!(value["status"], "success"); + } +} + +#[test] +fn llvm_emission_receives_the_actual_crate_codegen_level() { + let temporary = Temporary::new(); + let (options, mut arguments) = setup(&temporary); + arguments.extend(args(&["-C", "opt-level=1"])); + let mut runner = FixtureRunner::new(); + let outcome = run_with(Path::new("rustc"), &arguments, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 0); + let call = runner + .calls + .iter() + .find(|c| c.program == Path::new("controller")) + .unwrap(); + let index = call + .arguments + .iter() + .position(|v| v == "--llvm-codegen-opt-level") + .expect("LLVM object emission ignored the actual crate optimization setting"); + assert_eq!(call.arguments[index + 1], OsString::from("1")); +} + +#[test] +fn unsupported_gpu_modules_use_explicit_cpu_fallback_without_claiming_hardware_execution() { + let temporary = Temporary::new(); + let (mut options, args) = setup(&temporary); + options.backend = "gpu".into(); + options.allow_fallback = true; + options.gpu = Some(GpuOptions { + emitter: "gpu-emitter".into(), + backend: "metal".into(), + library: "metal-library".into(), + shader: Some("metal-shader".into()), + device: 0, + }); + let mut runner = FixtureRunner::new(); + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 0); + assert!(!outcome.report.gpu_executed); + let unit = &outcome.report.units[0]; + assert_eq!(unit.backend, "llvm"); + assert!(!unit.physical_gpu); + assert_eq!(unit.kernel_dispatches, 0); + assert!(unit + .fallback_reason + .as_ref() + .unwrap() + .contains("unsupported globals")); + assert!(runner + .calls + .iter() + .all(|call| call.program != Path::new("gpu-emitter"))); +} + +#[test] +fn disallowed_gpu_fallback_errors_before_linking_or_publishing_partial_outputs() { + let temporary = Temporary::new(); + let (mut options, args) = setup(&temporary); + options.backend = "gpu".into(); + options.gpu = Some(GpuOptions { + emitter: "gpu-emitter".into(), + backend: "metal".into(), + library: "metal-library".into(), + shader: None, + device: 0, + }); + let mut runner = FixtureRunner::new(); + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_ne!(outcome.exit_code, 0); + assert!(!outcome.report.gpu_executed); + assert!(outcome + .report + .error + .unwrap() + .contains("unsupported globals")); + assert!(runner + .calls + .iter() + .all(|call| call.program != Path::new("controller"))); + assert!(!temporary.0.join("deps/app").exists()); +} + +#[test] +fn foreign_objects_in_the_shared_output_directory_are_never_scanned_or_modified() { + let temporary = Temporary::new(); + let (options, args) = setup(&temporary); + let foreign = temporary.0.join("deps/foreign.rcgu.o"); + fs::write(&foreign, b"foreign-original").unwrap(); + let mut runner = FixtureRunner::new(); + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 0); + assert_eq!(outcome.report.units.len(), 1); + assert_eq!(fs::read(foreign).unwrap(), b"foreign-original"); +} + +#[test] +fn frontend_failure_preserves_exit_status_and_does_not_call_an_object_adapter() { + let temporary = Temporary::new(); + let (options, args) = setup(&temporary); + let mut runner = FixtureRunner::new(); + runner.frontend_exit = 17; + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 17); + assert_eq!(outcome.report.status, "failed"); + assert!(outcome.report.units.is_empty()); + assert_eq!(runner.calls.len(), 2); + assert!(outcome.report.staging_directory.unwrap().exists()); +} + +#[test] +fn accidental_native_codegen_is_rejected_instead_of_shadow_compilation() { + let temporary = Temporary::new(); + let (options, args) = setup(&temporary); + let mut runner = FixtureRunner::new(); + runner.native_first = true; + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_ne!(outcome.exit_code, 0); + assert!(outcome + .report + .error + .unwrap() + .contains("native object before")); + assert!(runner + .calls + .iter() + .all(|c| c.program != Path::new("controller"))); +} + +#[test] +fn backend_failure_never_links_partial_objects_or_overwrites_an_existing_executable() { + let temporary = Temporary::new(); + let (options, args) = setup(&temporary); + let mut runner = FixtureRunner::new(); + runner.controller_exit = 3; + let artifact = temporary.0.join("deps/app"); + fs::write(&artifact, b"prior executable").unwrap(); + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_ne!(outcome.exit_code, 0); + assert!(outcome + .report + .error + .unwrap() + .contains("object adapter failed")); + assert!(runner + .calls + .iter() + .all(|c| !c.arguments.iter().any(|v| v == "-Zlink-only"))); + assert_eq!(fs::read(artifact).unwrap(), b"prior executable"); +} + +#[test] +fn link_failure_preserves_the_real_link_exit_code_and_retains_owned_recovery_files() { + let temporary = Temporary::new(); + let (options, args) = setup(&temporary); + let mut runner = FixtureRunner::new(); + runner.link_exit = 23; + let outcome = run_with(Path::new("rustc"), &args, &options, &mut runner).unwrap(); + assert_eq!(outcome.exit_code, 23); + assert_eq!(outcome.report.status, "failed"); + assert_eq!(outcome.report.units.len(), 1); + assert!(outcome.report.staging_directory.unwrap().exists()); + assert!(!temporary.0.join("deps/app").exists()); +} + +#[test] +fn metadata_only_calls_inherit_original_io_and_never_set_bootstrap_or_claim_an_adapter() { + let temporary = Temporary::new(); + let (options, _) = setup(&temporary); + let mut runner = FixtureRunner::new(); + let outcome = run_with( + Path::new("rustc"), + &args(&["source.rs", "--emit=metadata", "--out-dir", "/tmp/deps"]), + &options, + &mut runner, + ) + .unwrap(); + assert_eq!(outcome.report.status, "passthrough"); + assert!(outcome.report.forwarded_passthrough); + assert!(outcome.report.units.is_empty()); + assert_eq!(runner.calls.len(), 1); + assert!(runner.calls[0].inherit_io); + assert!(!runner.calls[0].bootstrap); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/cargo_policy.rs b/experiments/gpu-rust-compiler/compiler/tests/cargo_policy.rs new file mode 100644 index 00000000000..240e0a61dc0 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/cargo_policy.rs @@ -0,0 +1,445 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs, + path::PathBuf, + process::Command, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; +static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0); + +struct Fixture(PathBuf); + +impl Fixture { + fn new() -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "gpu-cargo-{}-{nonce}-{}", + std::process::id(), + NEXT_FIXTURE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(root.join("src")).unwrap(); + fs::write( + root.join("Cargo.toml"), + concat!( + "[package]\nname='cargo-policy-fixture'\nversion='0.1.0'\nedition='2021'\n", + "[workspace]\n[profile.release]\noverflow-checks=true\npanic='unwind'\n", + ), + ) + .unwrap(); + fs::write(root.join("src/main.rs"), concat!( + "use std::sync::atomic::{AtomicUsize, Ordering};\n", + "static DROPS: AtomicUsize = AtomicUsize::new(0);\n", + "struct Guard; impl Drop for Guard { fn drop(&mut self) { DROPS.fetch_add(1, Ordering::Relaxed); } }\n", + "fn generic>(value: T) -> u64 { value.into() + 1 }\n", + "fn main() { std::panic::set_hook(Box::new(|_| {}));\n", + "let caught=std::panic::catch_unwind(|| { let _guard=Guard; let v=std::hint::black_box(u8::MAX); let _=v+1; });\n", + "assert!(caught.is_err()); assert_eq!(DROPS.load(Ordering::Relaxed),1);\n", + "let value=std::thread::spawn(|| generic(41u32)).join().unwrap(); println!(\"value={value} drops=1\"); }\n", + )).unwrap(); + fs::write( + root.join("Cargo.lock"), + "version = 4\n\n[[package]]\nname = \"cargo-policy-fixture\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + Self(root) + } + + fn driver(&self, action: &str, quality: &str) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_gpu-cargo")); + command + .args([action, "--manifest-path"]) + .arg(self.0.join("Cargo.toml")); + command.args([ + "--package", + "cargo-policy-fixture", + "--bin", + "cargo-policy-fixture", + "--quality", + quality, + "--target-root", + ]); + command.arg(self.0.join("owned-targets")).arg("--offline"); + command + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[test] +fn plan_selects_explicit_quality_and_does_not_build_or_create_target_directories() { + let fixture = Fixture::new(); + let output = fixture.driver("plan", "fast").output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["quality"], "fast"); + let arguments = report["cargo_arguments"].as_array().unwrap(); + for config in [ + "profile.release.opt-level=1", + "profile.release.codegen-units=16", + "profile.release.lto=\"off\"", + ] { + assert!(arguments.iter().any(|value| value == config), "{report}"); + } + assert!(report["actual_backend"].is_null()); + assert_eq!(report["gpu_compilation_jobs"], 0); + assert!(!fixture.0.join("owned-targets").exists()); +} + +fn json(output: &std::process::Output) -> serde_json::Value { + serde_json::from_slice(&output.stdout) + .unwrap_or_else(|error| panic!("{error}: {}", String::from_utf8_lossy(&output.stdout))) +} + +#[test] +fn quality_modes_build_complete_rust_programs_and_preserve_panic_and_overflow_settings() { + let fixture = Fixture::new(); + for quality in ["fast", "balanced", "release"] { + let report_path = fixture.0.join(format!("{quality}-report.json")); + let output = fixture + .driver("build", quality) + .arg("--report") + .arg(&report_path) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report = json(&output); + assert_eq!(report["status"], "ok"); + assert_eq!(report["quality"], quality); + assert_eq!(report["toolchain"], "1.98.1"); + assert_eq!(report["actual_backend"], "rustc-llvm-cpu"); + assert_eq!(report["gpu_compilation_jobs"], 0); + assert!(report["cargo_elapsed_ms"].as_f64().unwrap() > 0.0); + let path = PathBuf::from(report["artifact"]["path"].as_str().unwrap()); + assert_eq!( + report["artifact"]["raw_bytes"].as_u64().unwrap(), + fs::metadata(&path).unwrap().len() + ); + assert_eq!( + report["artifact"]["raw_bytes"], + report["artifact"]["measured_bytes"] + ); + let execution = Command::new(&path).output().unwrap(); + assert!( + execution.status.success(), + "{}", + String::from_utf8_lossy(&execution.stderr) + ); + assert_eq!(execution.stdout, b"value=42 drops=1\n"); + let saved: serde_json::Value = + serde_json::from_slice(&fs::read(&report_path).unwrap()).unwrap(); + assert_eq!(saved, report); + assert!(fixture + .0 + .join("owned-targets") + .join(quality) + .join("native") + .join(".gpu-cargo-owner.json") + .exists()); + } +} + +#[test] +fn required_gpu_placement_fails_before_cargo_or_target_creation() { + let fixture = Fixture::new(); + for placement in ["metal", "cuda"] { + let output = fixture + .driver("build", "fast") + .args(["--placement", placement]) + .output() + .unwrap(); + assert!(!output.status.success()); + let report = json(&output); + assert_eq!(report["status"], "error"); + assert!(report["error"] + .as_str() + .unwrap() + .contains("GPU placement is unavailable")); + } + assert!(!fixture.0.join("owned-targets").exists()); +} + +#[test] +fn driver_refuses_to_adopt_unowned_target_contents() { + let fixture = Fixture::new(); + let target = fixture.0.join("owned-targets/fast/native"); + fs::create_dir_all(&target).unwrap(); + fs::write(target.join("other-work"), "preserve me").unwrap(); + let output = fixture.driver("build", "fast").output().unwrap(); + assert!(!output.status.success()); + assert!(json(&output)["error"] + .as_str() + .unwrap() + .contains("Refusing to adopt")); + assert_eq!( + fs::read_to_string(target.join("other-work")).unwrap(), + "preserve me" + ); + assert!(!target.join(".gpu-cargo-owner.json").exists()); +} + +#[test] +fn size_limit_compares_complete_files_and_fails_without_claiming_success() { + let fixture = Fixture::new(); + let first = fixture.driver("build", "fast").output().unwrap(); + assert!(first.status.success()); + let report = json(&first); + let artifact = PathBuf::from(report["artifact"]["path"].as_str().unwrap()); + let baseline = fixture.0.join("reference"); + fs::copy(&artifact, &baseline).unwrap(); + let second = fixture + .driver("build", "fast") + .arg("--reference-artifact") + .arg(&baseline) + .args(["--max-size-ratio", "1.0"]) + .output() + .unwrap(); + assert!( + second.status.success(), + "{}", + String::from_utf8_lossy(&second.stderr) + ); + let second = json(&second); + assert_eq!(second["artifact"]["reference"]["observed_ratio"], 1.0); + assert_eq!(second["artifact"]["reference"]["within_limit"], true); + assert_eq!(second["artifact"]["cargo_fresh"], true); + fs::write(&baseline, b"declared reference").unwrap(); + let failed = fixture + .driver("build", "fast") + .arg("--reference-artifact") + .arg(&baseline) + .output() + .unwrap(); + assert!(!failed.status.success()); + let failed = json(&failed); + assert_eq!(failed["status"], "size-limit-exceeded"); + assert_eq!(failed["artifact"]["reference"]["measured_bytes"], 18); + assert_eq!(failed["artifact"]["reference"]["within_limit"], false); + assert!(artifact.exists()); +} + +#[test] +fn same_strip_basis_measures_copies_and_preserves_original_artifacts() { + if !cfg!(any(target_os = "macos", target_os = "linux")) { + return; + } + let fixture = Fixture::new(); + let first = fixture.driver("build", "fast").output().unwrap(); + assert!(first.status.success()); + let first = json(&first); + let artifact = PathBuf::from(first["artifact"]["path"].as_str().unwrap()); + let original = fs::read(&artifact).unwrap(); + let baseline = fixture.0.join("reference"); + fs::write(&baseline, &original).unwrap(); + let output = fixture + .driver("build", "fast") + .arg("--reference-artifact") + .arg(&baseline) + .args(["--size-basis", "strip-debug", "--max-size-ratio", "1.0"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report = json(&output); + assert_eq!(report["artifact"]["size_basis"], "strip-debug"); + assert_eq!(report["artifact"]["reference"]["observed_ratio"], 1.0); + assert_eq!(fs::read(&artifact).unwrap(), original); + assert_eq!(fs::read(&baseline).unwrap(), original); +} + +#[test] +fn cargo_failure_reports_no_usable_artifact() { + let fixture = Fixture::new(); + fs::write( + fixture.0.join("src/main.rs"), + "compile_error!(\"intentional fixture failure\"); fn main() {}\n", + ) + .unwrap(); + let output = fixture.driver("build", "fast").output().unwrap(); + assert!(!output.status.success()); + let report = json(&output); + assert_eq!(report["status"], "build-failed"); + assert!(report["artifact"].is_null()); + assert!(report["error"].as_str().unwrap().contains("Cargo exited")); +} + +#[test] +fn report_cannot_replace_source_or_an_unrelated_existing_json_file() { + let fixture = Fixture::new(); + for path in [ + fixture.0.join("src/main.rs"), + fixture.0.join("important.json"), + ] { + if !path.exists() { + fs::write(&path, "{\"important\":true}\n").unwrap(); + } + let original = fs::read(&path).unwrap(); + let output = fixture + .driver("plan", "fast") + .arg("--report") + .arg(&path) + .output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(fs::read(&path).unwrap(), original); + } + assert!(!fixture.0.join("owned-targets").exists()); +} + +fn backend_tools(fixture: &Fixture) -> [PathBuf; 4] { + ["wrapper", "controller", "bridge", "llc"].map(|name| { + let path = fixture.0.join(name); + fs::write(&path, format!("declared {name} revision one")).unwrap(); + path + }) +} + +fn bitcode_plan(fixture: &Fixture, backend: &str, tools: &[PathBuf; 4]) -> Command { + let mut command = fixture.driver("plan", "fast"); + command + .args(["--object-backend", backend, "--bitcode-wrapper"]) + .arg(&tools[0]); + command.arg("--tpde-controller").arg(&tools[1]); + command.arg("--tpde-bridge").arg(&tools[2]); + command.arg("--llvm-llc").arg(&tools[3]); + command +} + +#[test] +fn bitcode_backend_plans_select_the_wrapper_and_isolate_its_content_provenance() { + let fixture = Fixture::new(); + let tools = backend_tools(&fixture); + let native = fixture.driver("plan", "fast").output().unwrap(); + assert!(native.status.success()); + let native = json(&native); + let tpde = bitcode_plan(&fixture, "tpde", &tools) + .arg("--allow-backend-fallback") + .output() + .unwrap(); + assert!( + tpde.status.success(), + "{}", + String::from_utf8_lossy(&tpde.stderr) + ); + let tpde = json(&tpde); + assert_eq!(tpde["object_backend"], "tpde"); + assert_eq!(tpde["backend_environment"]["GPU_LINK_BACKEND"], "tpde"); + assert_eq!(tpde["backend_environment"]["GPU_LINK_ALLOW_FALLBACK"], "1"); + let llvm_library = &tpde["backend_toolchain"]["rust_llvm_library"]; + assert_eq!(llvm_library["sha256"].as_str().unwrap().len(), 64); + assert_eq!( + tpde["backend_environment"]["RUST_LLVM_LIBRARY"], + llvm_library["path"] + ); + assert_eq!( + tpde["backend_toolchain"]["fallback_library_selection_verified"], + false + ); + assert!(tpde["actual_backend"].is_null()); + assert_ne!(native["target_dir"], tpde["target_dir"]); + fs::write(&tools[0], "declared wrapper revision two").unwrap(); + let changed = bitcode_plan(&fixture, "tpde", &tools) + .arg("--allow-backend-fallback") + .output() + .unwrap(); + assert!(changed.status.success()); + assert_ne!(tpde["target_dir"], json(&changed)["target_dir"]); + let llvm = bitcode_plan(&fixture, "llvm-bitcode", &tools) + .output() + .unwrap(); + assert!(llvm.status.success()); + assert_eq!( + json(&llvm)["backend_environment"]["GPU_LINK_BACKEND"], + "llvm" + ); + assert!(!fixture.0.join("owned-targets").exists()); +} + +#[test] +fn alternate_object_backends_reject_release_policy_without_building() { + let fixture = Fixture::new(); + let tools = backend_tools(&fixture); + let output = bitcode_plan(&fixture, "tpde", &tools) + .args(["--quality", "release"]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(json(&output)["error"] + .as_str() + .unwrap() + .contains("fast or balanced")); + assert!(!fixture.0.join("owned-targets").exists()); +} + +#[test] +fn required_gpu_plan_records_tools_without_claiming_hardware_execution() { + let fixture = Fixture::new(); + let tools = backend_tools(&fixture); + let output = bitcode_plan(&fixture, "gpu", &tools) + .args(["--placement", "metal", "--gpu-emitter"]) + .arg(&tools[1]) + .arg("--gpu-library") + .arg(&tools[2]) + .arg("--gpu-shader") + .arg(&tools[3]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report = json(&output); + assert_eq!(report["object_backend"], "gpu"); + assert_eq!( + report["backend_environment"]["GPU_LINK_GPU_BACKEND"], + "metal" + ); + assert_eq!(report["backend_environment"]["GPU_LINK_GPU_DEVICE"], "0"); + assert!(report["actual_backend"].is_null()); + assert_eq!(report["gpu_compilation_jobs"], 0); + assert_eq!(report["gpu_executed"], false); + assert_eq!(report["current_run_gpu_qualified"], false); + assert_eq!(report["performance_speedup_demonstrated"], false); + assert!(!fixture.0.join("owned-targets").exists()); +} + +#[test] +fn inherited_llvm_library_override_cannot_silently_reuse_backend_provenance() { + let fixture = Fixture::new(); + let tools = backend_tools(&fixture); + let output = bitcode_plan(&fixture, "llvm-bitcode", &tools) + .env("RUST_LLVM_LIBRARY", &tools[2]) + .output() + .unwrap(); + assert!( + !output.status.success(), + "Library override was accepted: {}", + String::from_utf8_lossy(&output.stdout) + ); + assert!(json(&output)["error"] + .as_str() + .unwrap() + .contains("RUST_LLVM_LIBRARY")); + assert!(!fixture.0.join("owned-targets").exists()); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/codegen_service_contract.rs b/experiments/gpu-rust-compiler/compiler/tests/codegen_service_contract.rs new file mode 100644 index 00000000000..2b8aff1d64a --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/codegen_service_contract.rs @@ -0,0 +1,862 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[path = "../src/codegen_service.rs"] +#[allow(dead_code)] +mod codegen_service; + +use codegen_service::*; +use std::{ + sync::{mpsc, Arc, Mutex}, + time::Duration, +}; + +fn batch(id: &str, generation: u64) -> Arc { + Arc::new( + FlatBatch::new( + BatchIdentity { + workload_id: "scalar-fixture".into(), + toolchain_id: "fixture-emitter-v1".into(), + policy_id: "stack-home-fast-v1".into(), + target_abi: "test-i64".into(), + source_revision: format!("source-{generation}"), + }, + BatchKey { + logical_id: id.into(), + generation, + }, + vec![Function { + symbol: 0, + instruction_start: 0, + instruction_count: 2, + value_count: 1, + abi: 0, + }], + vec![ + Instruction { + opcode: Opcode::Const, + result: Some(0), + operands: [0, 0], + immediate: 42, + }, + Instruction { + opcode: Opcode::Return, + result: None, + operands: [0, 0], + immediate: 0, + }, + ], + vec![Symbol { + name: "answer".into(), + }], + vec![Abi { + calling_convention: CallingConvention::SystemV, + parameter_count: 0, + returns_i64: true, + requires_unwind: false, + }], + ) + .unwrap(), + ) +} + +fn cpu() -> Box { + Box::new(CpuCallbackExecutor::new("cpu-0", |input| { + Ok(Emission { + functions: input + .functions() + .iter() + .map(|f| FunctionEmission { + symbol: f.symbol, + code: input.instructions()[f.instruction_start as usize] + .immediate + .to_le_bytes() + .to_vec(), + relocations: vec![], + unwind: vec![], + }) + .collect(), + execution: ExecutionRecord::Cpu, + }) + })) +} + +fn output(input: &FlatBatch, execution: ExecutionRecord) -> Emission { + Emission { + functions: input + .functions() + .iter() + .map(|f| FunctionEmission { + symbol: f.symbol, + code: input.instructions()[f.instruction_start as usize] + .immediate + .to_le_bytes() + .to_vec(), + relocations: vec![], + unwind: vec![], + }) + .collect(), + execution, + } +} + +struct ReleaseOnDrop(Option>); +impl ReleaseOnDrop { + fn release(mut self) { + self.0.take().unwrap().send(()).unwrap(); + } +} +impl Drop for ReleaseOnDrop { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } +} + +type GatedCpu = ( + Box, + mpsc::Receiver, + mpsc::Sender<()>, + Arc>>, +); +fn gated_cpu() -> GatedCpu { + let (started, observed) = mpsc::channel(); + let (release, resumed) = mpsc::channel(); + let resumed = Mutex::new(resumed); + let order = Arc::new(Mutex::new(Vec::new())); + let recorded = order.clone(); + let executor = CpuCallbackExecutor::new("cpu-0", move |input| { + recorded + .lock() + .unwrap() + .push(input.key().logical_id.clone()); + if input.key().logical_id == "gate" { + started.send(input.key().logical_id.clone()).unwrap(); + resumed.lock().unwrap().recv().unwrap(); + } + Ok(output(input, ExecutionRecord::Cpu)) + }); + (Box::new(executor), observed, release, order) +} + +fn wait_started(receiver: &mpsc::Receiver) { + assert_eq!( + receiver.recv_timeout(Duration::from_secs(5)).unwrap(), + "gate" + ); +} + +#[test] +fn cpu_callback_outputs_are_bound_to_the_submitted_batch_and_never_gpu_proof() { + let service = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let client = service.register_client("rustc-a", 1).unwrap(); + let input = batch("answer", 0); + let ticket = service + .submit(&client, input.clone(), SubmitOptions::default()) + .unwrap(); + let receipt = ticket.wait().unwrap(); + assert_eq!(receipt.batch_key, input.key().clone()); + assert_eq!(receipt.identity, input.identity().clone()); + assert_eq!(receipt.output.functions[0].code, 42_i64.to_le_bytes()); + assert!(!receipt.execution.is_physical_gpu()); + assert_eq!(receipt.route.device_id, "cpu-0"); + assert_eq!(service.client_credits(&client).unwrap(), (1, 1)); +} + +#[test] +fn requesting_cuda_without_a_physical_cuda_executor_returns_an_error() { + let service = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let client = service.register_client("rustc-a", 1).unwrap(); + let options = SubmitOptions { + placement: Placement::RequireGpu(GpuBackend::Cuda), + ..Default::default() + }; + assert!(matches!( + service.submit(&client, batch("cuda", 0), options), + Err(ServiceError::Unavailable(_)) + )); +} + +#[test] +fn undefined_values_are_rejected_before_executor_submission() { + let original = batch("invalid", 0); + let mut instructions = original.instructions().to_vec(); + instructions[0].opcode = Opcode::Copy; + let error = FlatBatch::new( + original.identity().clone(), + original.key().clone(), + original.functions().to_vec(), + instructions, + original.symbols().to_vec(), + original.abis().to_vec(), + ) + .unwrap_err(); + assert!(error.contains("undefined"), "{error}"); +} + +#[test] +fn queue_backpressure_does_not_replace_or_cancel_an_existing_request() { + let (executor, started, release, _) = gated_cpu(); + let config = SessionConfig { + max_queued_batches: 1, + ..Default::default() + }; + let service = BuildSession::new(config, vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let gate = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + let queued = service + .submit(&client, batch("work", 0), SubmitOptions::default()) + .unwrap(); + assert!(matches!( + service.submit(&client, batch("work", 1), SubmitOptions::default()), + Err(ServiceError::Backpressure) + )); + assert_eq!(queued.status(), JobStatus::Queued); + release.release(); + gate.wait().unwrap(); + assert_eq!(queued.wait().unwrap().batch_key.generation, 0); +} + +#[test] +fn owned_input_budget_includes_running_work_and_rejects_oversize_submissions() { + let input = batch("gate", 0); + let config = SessionConfig { + max_owned_bytes: input.owned_bytes(), + ..Default::default() + }; + let (executor, started, release, _) = gated_cpu(); + let service = BuildSession::new(config, vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let gate = service + .submit(&client, input.clone(), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + assert_eq!(service.snapshot().owned_bytes, input.owned_bytes()); + assert!(matches!( + service.submit(&client, batch("work", 0), SubmitOptions::default()), + Err(ServiceError::Backpressure) + )); + release.release(); + gate.wait().unwrap(); + assert_eq!(service.snapshot().owned_bytes, 0); +} + +#[test] +fn cancellation_retains_input_and_client_credit_until_pending_work_completes() { + let (executor, started, release, _) = gated_cpu(); + let service = BuildSession::new(SessionConfig::default(), vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let input = batch("gate", 0); + let weak = Arc::downgrade(&input); + let ticket = service + .submit(&client, input.clone(), SubmitOptions::default()) + .unwrap(); + drop(input); + wait_started(&started); + assert_eq!( + ticket.cancel(), + Cancellation { + was_running: true, + resources_pending: true + } + ); + assert_eq!(ticket.status(), JobStatus::CancellationPending); + assert!(weak.upgrade().is_some()); + assert_eq!(service.client_credits(&client).unwrap(), (1, 0)); + assert!(service.snapshot().owned_bytes > 0); + release.release(); + assert!(matches!(ticket.wait(), Err(ServiceError::Canceled))); + assert!(weak.upgrade().is_none()); + assert_eq!(service.client_credits(&client).unwrap(), (1, 1)); + assert_eq!(service.snapshot().owned_bytes, 0); +} + +#[test] +fn newer_source_generation_suppresses_a_running_old_generation() { + let (executor, started, release, _) = gated_cpu(); + let service = BuildSession::new(SessionConfig::default(), vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let old = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + let new = service + .submit(&client, batch("gate", 1), SubmitOptions::default()) + .unwrap(); + assert_eq!(old.status(), JobStatus::CancellationPending); + assert!(matches!( + service.submit(&client, batch("gate", 0), SubmitOptions::default()), + Err(ServiceError::Duplicate) + )); + // The callback gates every revision. Permit both to finish. + release.0.as_ref().unwrap().send(()).unwrap(); + assert!(matches!(old.wait(), Err(ServiceError::Canceled))); + wait_started(&started); + release.release(); + assert_eq!(new.wait().unwrap().batch_key.generation, 1); +} + +#[test] +fn queued_cancellation_releases_input_without_executor_submission() { + let (executor, started, release, order) = gated_cpu(); + let service = BuildSession::new(SessionConfig::default(), vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let gate = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + let input = batch("never-executed", 0); + let weak = Arc::downgrade(&input); + let ticket = service + .submit(&client, input, SubmitOptions::default()) + .unwrap(); + assert_eq!( + ticket.cancel(), + Cancellation { + was_running: false, + resources_pending: false + } + ); + assert!(matches!(ticket.wait(), Err(ServiceError::Canceled))); + assert!(weak.upgrade().is_none()); + release.release(); + gate.wait().unwrap(); + assert_eq!(*order.lock().unwrap(), ["gate"]); +} + +#[test] +fn prerequisites_override_priority_and_ready_critical_work_runs_first() { + let (executor, started, release, order) = gated_cpu(); + let service = BuildSession::new(SessionConfig::default(), vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let gate = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + let low = service + .submit(&client, batch("low", 0), SubmitOptions::default()) + .unwrap(); + let critical = service + .submit( + &client, + batch("critical", 0), + SubmitOptions { + critical_path_cost: 100, + ..Default::default() + }, + ) + .unwrap(); + let dependent = service + .submit( + &client, + batch("dependent", 0), + SubmitOptions { + prerequisites: vec![low.key().clone()], + critical_path_cost: 1000, + ..Default::default() + }, + ) + .unwrap(); + release.release(); + for ticket in [gate, low, critical, dependent] { + ticket.wait().unwrap(); + } + assert_eq!( + *order.lock().unwrap(), + ["gate", "critical", "low", "dependent"] + ); +} + +#[test] +fn aging_dispatches_old_low_priority_work_before_an_unbounded_high_priority_stream() { + let (executor, started, release, order) = gated_cpu(); + let service = BuildSession::new( + SessionConfig { + starvation_after_dispatches: 2, + ..Default::default() + }, + vec![executor], + ) + .unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let gate = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + let mut tickets = vec![ + gate, + service + .submit(&client, batch("old", 0), SubmitOptions::default()) + .unwrap(), + ]; + for (id, priority) in [("high-3", 3), ("high-2", 2), ("high-1", 1)] { + tickets.push( + service + .submit( + &client, + batch(id, 0), + SubmitOptions { + critical_path_cost: priority, + ..Default::default() + }, + ) + .unwrap(), + ); + } + release.release(); + for ticket in tickets { + ticket.wait().unwrap(); + } + assert_eq!( + *order.lock().unwrap(), + ["gate", "high-3", "high-2", "old", "high-1"] + ); +} + +#[test] +fn canceled_prerequisites_fail_dependents_without_running_them() { + let (executor, started, release, order) = gated_cpu(); + let service = BuildSession::new(SessionConfig::default(), vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let gate = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + let canceled = service + .submit(&client, batch("prerequisite", 0), SubmitOptions::default()) + .unwrap(); + let dependent = service + .submit( + &client, + batch("dependent", 0), + SubmitOptions { + prerequisites: vec![canceled.key().clone()], + ..Default::default() + }, + ) + .unwrap(); + canceled.cancel(); + release.release(); + gate.wait().unwrap(); + assert!(matches!( + dependent.wait(), + Err(ServiceError::DependencyFailed) + )); + assert_eq!(*order.lock().unwrap(), ["gate"]); +} + +struct FixedExecutor { + capabilities: DeviceCapabilities, + execution: ExecutionRecord, +} +struct FixedPending { + batch: Arc, + execution: ExecutionRecord, +} +impl PendingExecution for FixedPending { + fn wait(self: Box) -> Result { + Ok(output(&self.batch, self.execution)) + } +} +impl BatchExecutor for FixedExecutor { + fn capabilities(&self) -> DeviceCapabilities { + self.capabilities.clone() + } + fn submit(&mut self, batch: Arc) -> Result, String> { + Ok(Box::new(FixedPending { + batch, + execution: self.execution.clone(), + })) + } +} + +#[test] +fn simulated_cuda_never_qualifies_required_cuda_and_reports_simulation() { + let simulation = FixedExecutor { + capabilities: DeviceCapabilities::simulated_gpu(GpuBackend::Cuda, "sim-0"), + execution: ExecutionRecord::Simulated { + backend: GpuBackend::Cuda, + device_id: "sim-0".into(), + }, + }; + let service = BuildSession::new(SessionConfig::default(), vec![Box::new(simulation)]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + assert!(matches!( + service.submit( + &client, + batch("required", 0), + SubmitOptions { + placement: Placement::RequireGpu(GpuBackend::Cuda), + ..Default::default() + } + ), + Err(ServiceError::Unavailable(_)) + )); + assert!(matches!( + service.submit(&client, batch("auto", 0), SubmitOptions::default()), + Err(ServiceError::Unavailable(_)) + )); + let ticket = service + .submit( + &client, + batch("sim", 0), + SubmitOptions { + placement: Placement::SimulatedGpu(GpuBackend::Cuda), + ..Default::default() + }, + ) + .unwrap(); + let receipt = ticket.wait().unwrap(); + assert_eq!(receipt.route.reality, ExecutionReality::Simulation); + assert!(!receipt.execution.is_physical_gpu()); + assert!(receipt.route.reason.contains("not GPU evidence")); +} + +#[test] +fn a_simulator_cannot_publish_a_forged_physical_execution_record() { + let simulation = FixedExecutor { + capabilities: DeviceCapabilities::simulated_gpu(GpuBackend::Cuda, "sim-0"), + execution: ExecutionRecord::PhysicalGpu { + backend: GpuBackend::Cuda, + device_id: "sim-0".into(), + kernel_dispatches: 1, + kernel_output_used: true, + }, + }; + let service = BuildSession::new(SessionConfig::default(), vec![Box::new(simulation)]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + let ticket = service + .submit( + &client, + batch("forged", 0), + SubmitOptions { + placement: Placement::SimulatedGpu(GpuBackend::Cuda), + ..Default::default() + }, + ) + .unwrap(); + assert!(matches!(ticket.wait(), Err(ServiceError::Output(_)))); +} + +#[test] +fn duplicate_device_ids_and_unfunded_cpu_workers_are_rejected() { + assert!(matches!( + BuildSession::new(SessionConfig::default(), vec![cpu(), cpu()]), + Err(ServiceError::Invalid(_)) + )); + let service = BuildSession::new( + SessionConfig { + cpu_credit_limit: 1, + ..Default::default() + }, + vec![cpu()], + ) + .unwrap(); + service.register_client("rustc-one", 1).unwrap(); + assert!(matches!( + service.register_client("rustc-two", 1), + Err(ServiceError::Invalid(_)) + )); +} + +#[test] +fn client_credentials_from_another_build_session_are_rejected() { + let first = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let second = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let client = first.register_client("rustc", 1).unwrap(); + second.register_client("rustc", 1).unwrap(); + assert!(matches!( + second.submit(&client, batch("other-session", 0), SubmitOptions::default()), + Err(ServiceError::UnknownClient) + )); +} + +#[test] +fn executor_failure_and_panic_return_credits_and_do_not_poison_later_submissions() { + let executor = + CpuCallbackExecutor::new("cpu-0", |input| match input.key().logical_id.as_str() { + "failure" => Err("test execution failed".into()), + "panic" => panic!("test callback panicked"), + _ => Ok(output(input, ExecutionRecord::Cpu)), + }); + let service = BuildSession::new(SessionConfig::default(), vec![Box::new(executor)]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + assert!(matches!( + service + .submit(&client, batch("failure", 0), SubmitOptions::default()) + .unwrap() + .wait(), + Err(ServiceError::Execution(_)) + )); + assert!(matches!( + service + .submit(&client, batch("panic", 0), SubmitOptions::default()) + .unwrap() + .wait(), + Err(ServiceError::WorkerPanicked) + )); + service + .submit(&client, batch("recovery", 0), SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); + assert_eq!(service.client_credits(&client).unwrap(), (1, 1)); +} + +#[test] +fn partial_output_and_missing_required_unwind_are_not_published() { + let executor = CpuCallbackExecutor::new("cpu-0", |input| { + let mut emission = output(input, ExecutionRecord::Cpu); + if input.key().logical_id == "partial" { + emission.functions.clear(); + } + Ok(emission) + }); + let service = BuildSession::new(SessionConfig::default(), vec![Box::new(executor)]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + assert!(matches!( + service + .submit(&client, batch("partial", 0), SubmitOptions::default()) + .unwrap() + .wait(), + Err(ServiceError::Output(_)) + )); + let original = batch("unwind", 0); + let mut abi = original.abis().to_vec(); + abi[0].requires_unwind = true; + let input = Arc::new( + FlatBatch::new( + original.identity().clone(), + original.key().clone(), + original.functions().to_vec(), + original.instructions().to_vec(), + original.symbols().to_vec(), + abi, + ) + .unwrap(), + ); + assert!(matches!( + service + .submit(&client, input, SubmitOptions::default()) + .unwrap() + .wait(), + Err(ServiceError::Output(_)) + )); +} + +#[test] +fn canonical_output_order_uses_stable_symbol_names_instead_of_worker_order() { + let original = batch("order", 0); + let mut functions = original.functions().to_vec(); + functions.push(Function { + symbol: 1, + instruction_start: 2, + instruction_count: 2, + value_count: 1, + abi: 0, + }); + let mut instructions = original.instructions().to_vec(); + instructions.extend(original.instructions().iter().cloned()); + let input = Arc::new( + FlatBatch::new( + original.identity().clone(), + original.key().clone(), + functions, + instructions, + vec![ + Symbol { + name: "zebra".into(), + }, + Symbol { name: "ant".into() }, + ], + original.abis().to_vec(), + ) + .unwrap(), + ); + let service = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + let receipt = service + .submit(&client, input, SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); + assert_eq!( + receipt + .output + .functions + .iter() + .map(|f| f.symbol) + .collect::>(), + [1, 0] + ); + assert_eq!(receipt.schema_version, 1); + assert!(!receipt.route.reason.is_empty()); +} + +#[test] +fn dropping_the_last_ticket_releases_completed_output_but_preserves_prerequisite_success() { + let service = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + let ticket = service + .submit(&client, batch("first", 0), SubmitOptions::default()) + .unwrap(); + let key = ticket.key().clone(); + let receipt = ticket.wait().unwrap(); + let weak = Arc::downgrade(&receipt); + drop(receipt); + assert!(weak.upgrade().is_some()); + drop(ticket); + assert!(weak.upgrade().is_none()); + service + .submit( + &client, + batch("dependent", 0), + SubmitOptions { + prerequisites: vec![key], + ..Default::default() + }, + ) + .unwrap() + .wait() + .unwrap(); +} + +#[test] +fn dropping_an_owner_while_running_cancels_publication_and_retains_storage_until_completion() { + let (executor, started, release, _) = gated_cpu(); + let service = BuildSession::new(SessionConfig::default(), vec![executor]).unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let input = batch("gate", 0); + let weak = Arc::downgrade(&input); + let ticket = service + .submit(&client, input, SubmitOptions::default()) + .unwrap(); + wait_started(&started); + drop(ticket); + assert!(weak.upgrade().is_some()); + assert_eq!(service.client_credits(&client).unwrap(), (1, 0)); + let after = service + .submit(&client, batch("after", 0), SubmitOptions::default()) + .unwrap(); + release.release(); + after.wait().unwrap(); + assert!(weak.upgrade().is_none()); +} + +#[test] +fn client_credits_are_reusable_after_the_client_has_no_pending_work() { + let service = BuildSession::new( + SessionConfig { + cpu_credit_limit: 1, + ..Default::default() + }, + vec![cpu()], + ) + .unwrap(); + let client = service.register_client("rustc-first", 1).unwrap(); + service + .submit(&client, batch("first", 0), SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); + service.unregister_client(&client).unwrap(); + assert!(matches!( + service.client_credits(&client), + Err(ServiceError::UnknownClient) + )); + let next = service.register_client("rustc-second", 1).unwrap(); + service + .submit(&next, batch("second", 0), SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); +} + +#[test] +fn completed_receipts_are_rejected_after_a_new_source_generation_and_in_other_sessions() { + let service = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let other = BuildSession::new(SessionConfig::default(), vec![cpu()]).unwrap(); + let client = service.register_client("rustc", 1).unwrap(); + let receipt = service + .submit(&client, batch("source", 0), SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); + assert!(service.is_current(&receipt)); + assert!(!other.is_current(&receipt)); + let next = service + .submit(&client, batch("source", 1), SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); + assert!(!service.is_current(&receipt)); + assert!(service.is_current(&next)); +} + +#[test] +fn an_old_client_handle_cannot_claim_credits_after_its_identifier_is_reused() { + let service = BuildSession::new( + SessionConfig { + cpu_credit_limit: 1, + ..Default::default() + }, + vec![cpu()], + ) + .unwrap(); + let old = service.register_client("rustc", 1).unwrap(); + service.unregister_client(&old).unwrap(); + let new = service.register_client("rustc", 1).unwrap(); + assert!(matches!( + service.submit(&old, batch("old-client", 0), SubmitOptions::default()), + Err(ServiceError::UnknownClient) + )); + service + .submit(&new, batch("new-client", 0), SubmitOptions::default()) + .unwrap() + .wait() + .unwrap(); +} + +#[test] +fn unregistering_a_client_with_pending_work_cannot_reissue_its_cpu_credit() { + let (executor, started, release, _) = gated_cpu(); + let service = BuildSession::new( + SessionConfig { + cpu_credit_limit: 1, + ..Default::default() + }, + vec![executor], + ) + .unwrap(); + let release = ReleaseOnDrop(Some(release)); + let client = service.register_client("rustc", 1).unwrap(); + let ticket = service + .submit(&client, batch("gate", 0), SubmitOptions::default()) + .unwrap(); + wait_started(&started); + assert!(matches!( + service.unregister_client(&client), + Err(ServiceError::Invalid(_)) + )); + assert!(matches!( + service.register_client("other", 1), + Err(ServiceError::Invalid(_)) + )); + release.release(); + ticket.wait().unwrap(); + service.unregister_client(&client).unwrap(); + service.register_client("other", 1).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/corpus_replay_contract.rs b/experiments/gpu-rust-compiler/compiler/tests/corpus_replay_contract.rs new file mode 100644 index 00000000000..d8077be95df --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/corpus_replay_contract.rs @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::PathBuf, + process::Command, + sync::atomic::{AtomicUsize, Ordering}, +}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "gpu-corpus-contract-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + Self(path) + } + fn rejected(&self) -> serde_json::Value { + let input = self.0.join("captured.bc"); + let bytes = b"test capture; never submitted as compiler input"; + fs::write(&input, bytes).unwrap(); + serde_json::json!({"schema":"gpu-rust-corpus-census","version":1,"corpus_id":"test-corpus","source_revision":"test-revision","toolchain_id":"rust1.98.1-llvm22.1.8","total_units":1,"eligible_units":0,"rejected_units":1,"rejection_reasons":{"unsupported module":1},"exhaustive_capture_verified":true,"units":[{"input_path":input,"input_sha256":Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect::(),"status":"rejected","reason":"unsupported module","eligibility_ms":0.2}]}) + } + fn run(&self, manifest: &serde_json::Value) -> std::process::Output { + let input = self.0.join("manifest.json"); + fs::write(&input, serde_json::to_vec(manifest).unwrap()).unwrap(); + Command::new(env!("CARGO_BIN_EXE_gpu-corpus-replay")) + .args([ + "--manifest", + input.to_str().unwrap(), + "--report", + self.0.join("report.json").to_str().unwrap(), + "--backend", + "metal", + "--library", + "/absent/native/library", + ]) + .output() + .unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[test] +fn zero_eligible_modules_stop_before_gpu_initialization() { + let fixture = Fixture::new(); + let output = fixture.run(&fixture.rejected()); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = + serde_json::from_slice(&fs::read(fixture.0.join("report.json")).unwrap()).unwrap(); + assert_eq!(report["decision"], "stop-zero-eligible-modules"); + assert_eq!(report["gpu_executed"], false); + assert_eq!(report["eligible_units"], 0); +} + +#[test] +fn an_unverified_or_modified_capture_cannot_publish_a_replay_report() { + let fixture = Fixture::new(); + let mut manifest = fixture.rejected(); + manifest["exhaustive_capture_verified"] = false.into(); + let output = fixture.run(&manifest); + assert!(!output.status.success()); + assert!(!fixture.0.join("report.json").exists()); + manifest["exhaustive_capture_verified"] = true.into(); + manifest["units"][0]["input_sha256"] = "00".repeat(32).into(); + let output = fixture.run(&manifest); + assert!(!output.status.success()); + assert!(!fixture.0.join("report.json").exists()); +} + +#[cfg(target_os = "macos")] +#[test] +#[ignore = "requires built TPDE bridge (GPU_CORPUS_BRIDGE) and a physical Metal device"] +fn captured_rust_modules_replay_with_persistent_services_and_stable_native_objects() { + let fixture = Fixture::new(); + let bridge = std::env::var_os("GPU_CORPUS_BRIDGE").expect("GPU_CORPUS_BRIDGE"); + let mut units = Vec::new(); + let sha = |bytes: &[u8]| { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect::() + }; + for index in 0..2 { + let source = fixture.0.join(format!("leaf{index}.rs")); + let input = fixture.0.join(format!("leaf{index}.bc")); + let packet = fixture.0.join(format!("leaf{index}.gem1")); + // Powers of two become LLVM shifts, outside the frozen scalar schema. + fs::write(&source, format!("#![no_std]\n#[no_mangle]\npub extern \"C\" fn replay_leaf_{index}(x:i64)->i64{{x.wrapping_mul({}).wrapping_add(11)}}\n",index*2+7)).unwrap(); + let compiled = Command::new("rustup") + .args([ + "run", + "1.98.1", + "rustc", + "--edition=2021", + "--crate-type=lib", + "--emit=llvm-bc", + "-Copt-level=1", + "-Cdebuginfo=0", + "-Cpanic=abort", + "-Ccodegen-units=1", + ]) + .arg(&source) + .arg("-o") + .arg(&input) + .output() + .unwrap(); + assert!( + compiled.status.success(), + "{}", + String::from_utf8_lossy(&compiled.stderr) + ); + let exported = Command::new(&bridge) + .arg("--export-leaf") + .arg(&input) + .arg(&packet) + .output() + .unwrap(); + assert!( + exported.status.success(), + "{}", + String::from_utf8_lossy(&exported.stderr) + ); + units.push(serde_json::json!({"input_path":input,"input_sha256":sha(&fs::read(&input).unwrap()),"packet_path":packet,"packet_sha256":sha(&fs::read(&packet).unwrap()),"status":"eligible","eligibility_ms":0.0})); + } + let manifest = fixture.0.join("manifest.json"); + fs::write(&manifest,serde_json::to_vec(&serde_json::json!({"schema":"gpu-rust-corpus-census","version":1,"corpus_id":"native-rust-test-corpus","source_revision":"native-test-fixture","toolchain_id":"rust1.98.1-llvm22.1.8","total_units":2,"eligible_units":2,"rejected_units":0,"rejection_reasons":{},"exhaustive_capture_verified":true,"units":units})).unwrap()).unwrap(); + let report_path = fixture.0.join("native-report.json"); + let shader = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("native/codegen.metal"); + let output = Command::new(env!("CARGO_BIN_EXE_gpu-corpus-replay")) + .arg("--manifest") + .arg(&manifest) + .arg("--report") + .arg(&report_path) + .args([ + "--backend", + "metal", + "--library", + env!("GPUEMIT_METAL_LIBRARY"), + ]) + .arg("--shader") + .arg(shader) + .args(["--repeats", "3", "--cpu-workers", "2"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report: serde_json::Value = + serde_json::from_slice(&fs::read(report_path).unwrap()).unwrap(); + assert_eq!(report["gpu_executed"], true); + assert_eq!(report["unique_replayed_units"], 2); + assert_eq!(report["persistent_sessions"], true); + assert_eq!(report["all_objects_stable"], true); + assert_eq!(report["whole_build_speedup_demonstrated"], false); + assert_eq!(report["rounds"].as_array().unwrap().len(), 3); + for round in report["rounds"].as_array().unwrap() { + for side in ["cpu", "gpu"] { + assert_eq!(round[side]["unique_modules"], 2); + for unit in round[side]["units"].as_array().unwrap() { + assert!(unit["object_bytes"].as_u64().unwrap() > 0); + assert_eq!(unit["physical_gpu"], side == "gpu"); + if side == "gpu" { + assert!(unit["kernel_dispatches"].as_u64().unwrap() > 0); + assert_eq!(unit["kernel_output_used"], true); + assert!(!unit["device_id"].as_str().unwrap().is_empty()); + } + } + } + } +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/cpu_liveness_cli.rs b/experiments/gpu-rust-compiler/compiler/tests/cpu_liveness_cli.rs new file mode 100644 index 00000000000..9a0fd39f031 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/cpu_liveness_cli.rs @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +#[test] +fn cpu_workpack_cli_emits_verified_results_and_repeated_variant_timings() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = + std::env::temp_dir().join(format!("gpu-cpu-liveness-{}-{nonce}", std::process::id())); + fs::create_dir(&root).unwrap(); + let input = root.join("input.bin"); + let output = root.join("output.bin"); + let words: Vec = vec![ + 1, 2, 4, 2, 2, 2, 0, 2, 0, 2, 0, 0, 2, 0, 2, 1, 0, 1, 2, 1, 0, 1, 0, 2, 8, 2, 0, 0, 0, 4, + 16, 0, 0, + ]; + let mut bytes = b"GLC1".to_vec(); + bytes.extend(words.iter().flat_map(|word| word.to_le_bytes())); + fs::write(&input, bytes).unwrap(); + let run = Command::new(env!("CARGO_BIN_EXE_gpu-cpu-liveness")) + .args([input.as_os_str(), output.as_os_str(), "2".as_ref()]) + .output() + .unwrap(); + assert!( + run.status.success(), + "{}", + String::from_utf8_lossy(&run.stderr) + ); + let report = String::from_utf8(run.stdout).unwrap(); + for field in [ + "\"backend\":\"cpu-native\"", + "\"repeat_outputs_equal\":true", + "\"serial\"", + "\"function_parallel\"", + "\"word_parallel\"", + "\"samples_ms\"", + ] { + assert!(report.contains(field), "Missing {field}: {report}"); + } + let result = fs::read(&output).unwrap(); + let mut expected = b"GLR1".to_vec(); + expected.extend( + [4u32, 5, 24, 7, 24] + .iter() + .flat_map(|word| word.to_le_bytes()), + ); + assert_eq!(result, expected); + for repeats in ["0", "10001", "-1", "bad"] { + let rejected = root.join(format!("rejected-{repeats}.bin")); + let run = Command::new(env!("CARGO_BIN_EXE_gpu-cpu-liveness")) + .args([input.as_os_str(), rejected.as_os_str(), repeats.as_ref()]) + .output() + .unwrap(); + assert!(!run.status.success(), "Accepted invalid repeats {repeats}"); + assert!(!rejected.exists()); + } + let original = fs::read(&input).unwrap(); + let same_path = Command::new(env!("CARGO_BIN_EXE_gpu-cpu-liveness")) + .args([input.as_os_str(), input.as_os_str(), "1".as_ref()]) + .output() + .unwrap(); + assert!(!same_path.status.success(), "Overwrote the input workpack"); + assert_eq!(fs::read(&input).unwrap(), original); + fs::remove_dir_all(root).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/gpu_codegen_provenance.rs b/experiments/gpu-rust-compiler/compiler/tests/gpu_codegen_provenance.rs new file mode 100644 index 00000000000..3c9d5cdd318 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/gpu_codegen_provenance.rs @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn a_relocated_benchmark_records_ci_identity_and_unknown_worktree_state() { + use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, + }; + let root = std::env::temp_dir().join(format!( + "gpuemit-provenance-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&root).unwrap(); + let sha = "1234567890abcdef1234567890abcdef12345678"; + let result = Command::new(env!("CARGO_BIN_EXE_gpu-codegen")) + .current_dir(&root) + .env("CANDIDATE_SHA", sha) + .args([ + "--backend", + "cpu", + "--functions", + "1", + "--batch-functions", + "1", + "--repeats", + "1", + "--cpu-workers", + "1", + "--out-dir", + "output", + ]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let report: serde_json::Value = + serde_json::from_slice(&fs::read(root.join("output/report.json")).unwrap()).unwrap(); + assert_eq!(report["source_revision"], sha); + assert_eq!(report["source_revision_origin"], "ci-environment-assertion"); + assert!(report["worktree_dirty"].is_null()); + fs::remove_dir_all(root).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/gpu_emit_module.rs b/experiments/gpu-rust-compiler/compiler/tests/gpu_emit_module.rs new file mode 100644 index 00000000000..6c49ec725a7 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/gpu_emit_module.rs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +#[test] +fn a_failed_receipt_write_does_not_publish_a_gpu_object() { + use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, + }; + let root = std::env::temp_dir().join(format!( + "gpu-object-publication-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&root).unwrap(); + let mut packet = b"GEM1".to_vec(); + let triple = b"aarch64-apple-darwin"; + packet.extend_from_slice(&(triple.len() as u32).to_le_bytes()); + packet.extend_from_slice(triple); + packet.extend_from_slice(&1u32.to_le_bytes()); + let name = b"gpu_publication_probe"; + packet.extend_from_slice(&(name.len() as u32).to_le_bytes()); + packet.extend_from_slice(name); + for field in [0u32, 0, 1, 2] { + packet.extend_from_slice(&field.to_le_bytes()); + } + for (op, result, a, imm) in [(1u32, 0u32, 0u32, 42i64), (6, u32::MAX, 0, 0)] { + for field in [op, result, a, 0] { + packet.extend_from_slice(&field.to_le_bytes()); + } + packet.extend_from_slice(&imm.to_le_bytes()); + } + let input = root.join("input.gem"); + let output = root.join("leaf.o"); + let report = root.join("missing/report.json"); + fs::write(&input, packet).unwrap(); + let shader = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("native/codegen.metal"); + let result = Command::new(env!("CARGO_BIN_EXE_gpu-emit-module")) + .arg("--input") + .arg(input) + .arg("--output") + .arg(&output) + .args([ + "--backend", + "metal", + "--library", + env!("GPUEMIT_METAL_LIBRARY"), + ]) + .arg("--shader") + .arg(shader) + .arg("--report") + .arg(&report) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!( + !output.exists(), + "failed GPU emission published an unqualified native object: {}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(!report.exists()); + fs::remove_dir_all(root).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/leaf_packet_contract.rs b/experiments/gpu-rust-compiler/compiler/tests/leaf_packet_contract.rs new file mode 100644 index 00000000000..ecc6277b528 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/leaf_packet_contract.rs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use gpu_rust_compiler::leaf_packet; + +fn word(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_le_bytes()); +} +fn string(bytes: &mut Vec, value: &str) { + word(bytes, value.len() as u32); + bytes.extend_from_slice(value.as_bytes()); +} +fn packet() -> Vec { + let mut bytes = b"GEM1".to_vec(); + string(&mut bytes, "arm64-apple-macosx15.0.0"); + word(&mut bytes, 1); + string(&mut bytes, "rust_leaf"); + for value in [1, 1, 1, 2] { + word(&mut bytes, value); + } + for value in [0, 0, 0, 0] { + word(&mut bytes, value); + } + bytes.extend_from_slice(&0i64.to_le_bytes()); + for value in [6, u32::MAX, 0, 0] { + word(&mut bytes, value); + } + bytes.extend_from_slice(&0i64.to_le_bytes()); + bytes +} + +#[test] +fn shared_decoder_preserves_the_llvm_target_and_required_unwind() { + let decoded = leaf_packet::decode(&packet()).expect("valid strict LLVM leaf packet"); + assert_eq!(decoded.llvm_target, "arm64-apple-macosx15.0.0"); + assert_eq!(decoded.batch.identity().target_abi, "aarch64-apple-darwin"); + assert!(decoded.batch.abis()[0].requires_unwind); + assert_eq!(decoded.batch.symbols()[0].name, "rust_leaf"); +} + +#[test] +fn trailing_or_truncated_packets_cannot_be_replayed() { + let mut bytes = packet(); + bytes.push(0); + assert!(leaf_packet::decode(&bytes).is_err()); + bytes.truncate(bytes.len() - 2); + assert!(leaf_packet::decode(&bytes).is_err()); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/machine_codegen.rs b/experiments/gpu-rust-compiler/compiler/tests/machine_codegen.rs new file mode 100644 index 00000000000..d4072057c82 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/machine_codegen.rs @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use gpu_rust_compiler::{codegen_service::*, machine_codegen::*}; +use std::{ + fs, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; +static NEXT: AtomicU64 = AtomicU64::new(1); + +fn fixture(target: &str) -> FlatBatch { + FlatBatch::new( + BatchIdentity { + workload_id: "native-arithmetic".into(), + toolchain_id: "rust1.98.1".into(), + policy_id: "fast-leaf-v1".into(), + target_abi: target.into(), + source_revision: "test".into(), + }, + BatchKey { + logical_id: "leaf".into(), + generation: 1, + }, + vec![Function { + symbol: 0, + instruction_start: 0, + instruction_count: 6, + value_count: 5, + abi: 0, + }], + vec![ + Instruction { + opcode: Opcode::Argument, + result: Some(0), + operands: [0, 0], + immediate: 0, + }, + Instruction { + opcode: Opcode::Const, + result: Some(1), + operands: [0, 0], + immediate: -7, + }, + Instruction { + opcode: Opcode::Mul, + result: Some(2), + operands: [0, 1], + immediate: 0, + }, + Instruction { + opcode: Opcode::Const, + result: Some(3), + operands: [0, 0], + immediate: 19, + }, + Instruction { + opcode: Opcode::Add, + result: Some(4), + operands: [2, 3], + immediate: 0, + }, + Instruction { + opcode: Opcode::Return, + result: None, + operands: [4, 0], + immediate: 0, + }, + ], + vec![Symbol { + name: "gpu_leaf".into(), + }], + vec![Abi { + calling_convention: if target.contains("apple") && target.starts_with("aarch64") { + CallingConvention::AppleAarch64 + } else { + CallingConvention::SystemV + }, + parameter_count: 1, + returns_i64: true, + requires_unwind: false, + }], + ) + .unwrap() +} + +#[test] +fn target_identity_must_match_the_machine_code_target() { + let input = fixture("x86_64-unknown-linux-gnu"); + assert!(pack(&input, MachineTarget::Aarch64).is_err()); +} + +#[test] +fn externally_visible_rust_definitions_remain_public_in_native_objects() { + use object::{Object as _, ObjectSymbol as _}; + for (triple, target) in [ + ("x86_64-unknown-linux-gnu", MachineTarget::X86_64), + ("aarch64-apple-darwin", MachineTarget::Aarch64), + ] { + let input = fixture(triple); + let emission = cpu_emit(&input, target).unwrap(); + let data = write_object(&input, target, &emission).unwrap(); + let object = object::File::parse(data.as_slice()).unwrap(); + let symbol = object + .symbols() + .find(|s| s.name().unwrap_or("").trim_start_matches('_') == "gpu_leaf") + .unwrap(); + assert_eq!(symbol.scope(), object::SymbolScope::Dynamic); + } +} + +#[test] +fn required_unwind_emits_metadata_for_fixed_native_frames() { + use object::{Object as _, ObjectSection as _}; + for (triple, target, section) in [ + ( + "x86_64-unknown-linux-gnu", + MachineTarget::X86_64, + ".eh_frame", + ), + ( + "aarch64-unknown-linux-gnu", + MachineTarget::Aarch64, + ".eh_frame", + ), + ( + "aarch64-apple-darwin", + MachineTarget::Aarch64, + "__compact_unwind", + ), + ("aarch64-apple-darwin", MachineTarget::Aarch64, "__eh_frame"), + ("x86_64-apple-darwin", MachineTarget::X86_64, "__eh_frame"), + ] { + let base = fixture(triple); + let mut abis = base.abis().to_vec(); + abis[0].requires_unwind = true; + let input = FlatBatch::new( + base.identity().clone(), + base.key().clone(), + base.functions().to_vec(), + base.instructions().to_vec(), + base.symbols().to_vec(), + abis, + ) + .unwrap(); + let emission = cpu_emit(&input, target).unwrap(); + assert!(!emission.functions[0].unwind.is_empty()); + let bytes = write_object(&input, target, &emission).unwrap(); + let object = object::File::parse(bytes.as_slice()).unwrap(); + assert!(!object + .section_by_name(section) + .unwrap() + .data() + .unwrap() + .is_empty()); + } +} + +#[test] +fn emitted_cpu_object_links_and_runs_wrapping_arithmetic() { + let target = MachineTarget::host().unwrap(); + let triple = target.host_triple(); + let input = fixture(triple); + let emission = cpu_emit(&input, target).unwrap(); + let bytes = write_object(&input, target, &emission).unwrap(); + let dir = std::env::temp_dir().join(format!( + "gpuemit-object-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&dir).unwrap(); + fs::write(dir.join("leaf.o"), bytes).unwrap(); + fs::write(dir.join("main.c"),"#include \n#include \n#include \nextern uint64_t gpu_leaf(uint64_t);\nint main(void){uint64_t inputs[]={0,1,UINT64_MAX,INT64_MAX,(uint64_t)INT64_MIN};for(int i=0;i<5;i++)printf(\"%\" PRIu64 \"\\n\",gpu_leaf(inputs[i]));return 0;}\n").unwrap(); + let out = Command::new("clang") + .arg(dir.join("main.c")) + .arg(dir.join("leaf.o")) + .arg("-o") + .arg(dir.join("probe")) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let output = Command::new(dir.join("probe")).output().unwrap(); + assert!(output.status.success()); + let expected = [0i64, 1, -1, i64::MAX, i64::MIN] + .iter() + .map(|v| format!("{}\n", v.wrapping_mul(-7).wrapping_add(19) as u64)) + .collect::(); + assert_eq!(String::from_utf8(output.stdout).unwrap(), expected); + fs::remove_dir_all(dir).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/native_cuda_contract.rs b/experiments/gpu-rust-compiler/compiler/tests/native_cuda_contract.rs new file mode 100644 index 00000000000..fcb5c31fe90 --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/native_cuda_contract.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs, + process::Command, + time::{SystemTime, UNIX_EPOCH}, +}; + +#[test] +fn requested_cuda_reports_unavailable_hardware_without_cpu_output() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "native-cuda-contract-{}-{nonce}", + std::process::id() + )); + fs::create_dir(&directory).unwrap(); + let source = directory.join("source.rs"); + let output = directory.join("program"); + fs::write(&source, "fn main()->i64 { return 19; }").unwrap(); + let result = Command::new(env!("CARGO_BIN_EXE_gpu-rust-compiler")) + .arg(&source) + .args([ + "--backend", + "cuda", + "--cuda-algorithm", + "sparse", + "--cuda-library", + ]) + .arg(directory.join("unavailable-cuda-library.so")) + .arg("--output") + .arg(&output) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!( + !output.exists(), + "Unavailable CUDA must not emit a CPU fallback executable" + ); + let error = String::from_utf8_lossy(&result.stderr); + assert!( + error.contains("CUDA"), + "Missing CUDA availability diagnostic: {error}" + ); + assert!( + !error.contains("Unsupported native backend"), + "The native compiler has no CUDA adapter: {error}" + ); + fs::remove_dir_all(directory).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/compiler/tests/native_liveness_worker.rs b/experiments/gpu-rust-compiler/compiler/tests/native_liveness_worker.rs new file mode 100644 index 00000000000..a9c2a1587dc --- /dev/null +++ b/experiments/gpu-rust-compiler/compiler/tests/native_liveness_worker.rs @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs, + io::{BufRead, BufReader, Write}, + process::{Command, Stdio}, + time::{SystemTime, UNIX_EPOCH}, +}; + +#[test] +fn persistent_native_worker_replaces_facts_and_recovers_from_errors_without_cuda_fallback() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("native-worker-{}-{nonce}", std::process::id())); + fs::create_dir(&root).unwrap(); + let input = root.join("input.bin"); + let output = root.join("output.bin"); + let broken = root.join("broken.bin"); + let words = [ + 1u32, 2, 4, 2, 2, 2, 0, 2, 0, 2, 0, 0, 2, 0, 2, 1, 0, 1, 2, 1, 0, 1, 0, 2, 8, 2, 0, 0, 0, + 4, 16, 0, 0, + ]; + let mut bytes = b"GLC1".to_vec(); + bytes.extend(words.iter().flat_map(|word| word.to_le_bytes())); + fs::write(&input, &bytes).unwrap(); + fs::write(&broken, b"GLC1").unwrap(); + let mut child = Command::new(env!("CARGO_BIN_EXE_gpu-native-benchmark")) + .args(["--serve", "--threads", "2"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = BufReader::new(child.stdout.take().unwrap()); + let mut request = |id: &str, backend: &str, source: &std::path::Path, residency: &str| { + writeln!( + stdin, + "{id}\t{backend}\tdense\t{}\t{}\t{residency}\tserial", + source.display(), + output.display() + ) + .unwrap(); + stdin.flush().unwrap(); + let mut response = String::new(); + stdout.read_line(&mut response).unwrap(); + response + }; + let first = request("1", "cpu", &input, "update"); + assert!(first.contains("\"status\":\"ok\""), "{first}"); + assert!(first.contains("\"cuda_context_creations\":0"), "{first}"); + let mut expected = b"GLR1".to_vec(); + expected.extend( + [4u32, 5, 24, 7, 24] + .iter() + .flat_map(|word| word.to_le_bytes()), + ); + assert_eq!(fs::read(&output).unwrap(), expected); + let malformed = request("2", "cpu", &broken, "update"); + assert!(malformed.contains("\"status\":\"error\""), "{malformed}"); + let unavailable = request("3", "cuda", &input, "update"); + assert!( + unavailable.contains("\"status\":\"error\""), + "{unavailable}" + ); + assert!( + unavailable.contains("CUDA library was not specified"), + "{unavailable}" + ); + let recovered = request("4", "cpu", &input, "resident"); + assert!(recovered.contains("\"status\":\"ok\""), "{recovered}"); + // All twelve bitset words (USE/DEF/PHIOUT) become zero. The prior live-in + // facts must disappear rather than becoming an incorrect cached solution. + let mut cleared = bytes.clone(); + let facts_begin = cleared.len() - 12 * 4; + cleared[facts_begin..].fill(0); + fs::write(&input, cleared).unwrap(); + let stale = request("5", "cpu", &input, "resident"); + assert!(stale.contains("\"status\":\"error\""), "{stale}"); + let changed = request("6", "cpu", &input, "update"); + assert!(changed.contains("\"status\":\"ok\""), "{changed}"); + let result = fs::read(&output).unwrap(); + assert_eq!(&result[8..], &[0; 16]); + writeln!(stdin, "quit").unwrap(); + drop(stdin); + assert!(child.wait().unwrap().success()); + fs::remove_dir_all(root).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/cuda/CMakeLists.txt b/experiments/gpu-rust-compiler/cuda/CMakeLists.txt new file mode 100644 index 00000000000..291c059e329 --- /dev/null +++ b/experiments/gpu-rust-compiler/cuda/CMakeLists.txt @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.18) +project(gpu_rust_liveness LANGUAGES CXX CUDA) + +add_executable(cuda-liveness liveness.cu) +set_target_properties(cuda-liveness PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED YES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED YES + CUDA_EXTENSIONS NO +) diff --git a/experiments/gpu-rust-compiler/cuda/liveness.cu b/experiments/gpu-rust-compiler/cuda/liveness.cu new file mode 100644 index 00000000000..42e48c7cb86 --- /dev/null +++ b/experiments/gpu-rust-compiler/cuda/liveness.cu @@ -0,0 +1,490 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// CUDA backend for the same exact block-liveness equations as liveness.metal. +// INPUT: GLC1, six little-endian uint32 counts, then the documented arrays. +// OUTPUT: GLR1, little-endian uint32 cell count, then live-in uint32 cells. +// GLC_FORMAT_CHECK_ONLY builds a host parser checker. It never runs liveness. + +#ifndef GLC_FORMAT_CHECK_ONLY +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Group { + std::uint32_t block_base; + std::uint32_t block_count; + std::uint32_t row_base; + std::uint32_t words; + std::uint32_t word; +}; +static_assert(sizeof(Group) == 5 * sizeof(std::uint32_t), "Group layout changed"); + +struct Input { + std::uint32_t functions, blocks, cells, groups, max_blocks, edges; + std::vector descriptors; + std::vector successor_offsets, successors, uses, defs, phi_out; +}; + +class Reader { +public: + explicit Reader(const std::string& path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) throw std::runtime_error("cannot open input: " + path); + const auto length = file.tellg(); + if (length < 0 || static_cast(length) > + std::numeric_limits::max()) { + throw std::runtime_error("invalid input length"); + } + data.resize(static_cast(length)); + file.seekg(0); + if (!data.empty() && !file.read(reinterpret_cast(data.data()), length)) { + throw std::runtime_error("failed reading input"); + } + } + + void magic(const char* expected) { + require(4); + if (!std::equal(data.begin() + position, data.begin() + position + 4, + reinterpret_cast(expected))) { + throw std::runtime_error("expected GLC1 input magic"); + } + position += 4; + } + + std::uint32_t word() { + require(4); + const auto value = std::uint32_t(data[position]) | + (std::uint32_t(data[position + 1]) << 8) | + (std::uint32_t(data[position + 2]) << 16) | + (std::uint32_t(data[position + 3]) << 24); + position += 4; + return value; + } + + std::vector words(std::uint64_t count) { + if (count > (data.size() - position) / 4) { + throw std::runtime_error("truncated input array"); + } + std::vector result(static_cast(count)); + for (auto& value : result) value = word(); + return result; + } + + std::size_t remaining() const { return data.size() - position; } + +private: + std::vector data; + std::size_t position = 0; + void require(std::size_t count) const { + if (count > data.size() - position) throw std::runtime_error("truncated input"); + } +}; + +static void validate(const Input& input) { + if (!input.functions || !input.blocks || !input.cells || !input.groups) { + throw std::runtime_error("empty workload dimensions"); + } + if (input.successor_offsets.front() != 0 || + input.successor_offsets.back() != input.edges) { + throw std::runtime_error("successor offsets do not span the edges array"); + } + for (std::size_t i = 1; i < input.successor_offsets.size(); ++i) { + if (input.successor_offsets[i] < input.successor_offsets[i - 1] || + input.successor_offsets[i] > input.edges) { + throw std::runtime_error("invalid successor offsets"); + } + } + + struct Shape { + std::uint32_t blocks, row, words; + std::vector seen; + }; + std::map functions; + std::uint32_t largest = 0; + for (const auto& group : input.descriptors) { + if (!group.block_count || !group.words || group.words > input.groups || + group.word >= group.words || + std::uint64_t(group.block_base) + group.block_count > input.blocks || + std::uint64_t(group.row_base) + std::uint64_t(group.block_count) * group.words > input.cells) { + throw std::runtime_error("invalid function-word group"); + } + largest = std::max(largest, group.block_count); + auto found = functions.find(group.block_base); + if (found == functions.end()) { + found = functions.emplace(group.block_base, Shape{ + group.block_count, group.row_base, group.words, + std::vector(group.words, false), + }).first; + } + auto& shape = found->second; + if (shape.blocks != group.block_count || shape.row != group.row_base || + shape.words != group.words || shape.seen[group.word]) { + throw std::runtime_error("duplicate or inconsistent function-word group"); + } + shape.seen[group.word] = true; + } + if (functions.size() != input.functions || largest != input.max_blocks) { + throw std::runtime_error("function count or maximum block count does not match descriptors"); + } + std::uint64_t next_block = 0, next_cell = 0; + for (const auto& entry : functions) { + const auto base = entry.first; + const auto& shape = entry.second; + if (base != next_block || shape.row != next_cell || + std::find(shape.seen.begin(), shape.seen.end(), false) != shape.seen.end()) { + throw std::runtime_error("function groups do not cover every block and output cell exactly once"); + } + const auto end = std::uint64_t(base) + shape.blocks; + for (std::uint64_t block = base; block < end; ++block) { + for (auto edge = input.successor_offsets[block]; + edge < input.successor_offsets[block + 1]; ++edge) { + const auto successor = input.successors[edge]; + if (successor < base || successor >= end) { + throw std::runtime_error("CFG successor escapes its owning function"); + } + } + } + next_block = end; + next_cell += std::uint64_t(shape.blocks) * shape.words; + } + if (next_block != input.blocks || next_cell != input.cells) { + throw std::runtime_error("function groups leave uncovered blocks or cells"); + } +} + +static Input read_input(const std::string& path) { + Reader reader(path); + reader.magic("GLC1"); + Input input; + input.functions = reader.word(); input.blocks = reader.word(); + input.cells = reader.word(); input.groups = reader.word(); + input.max_blocks = reader.word(); input.edges = reader.word(); + const std::uint64_t expected_words = std::uint64_t(input.groups) * 5 + + std::uint64_t(input.blocks) + 1 + input.edges + std::uint64_t(input.cells) * 3; + if (expected_words * 4 != reader.remaining()) { + throw std::runtime_error("binary length does not match header dimensions"); + } + input.descriptors.resize(input.groups); + for (auto& group : input.descriptors) { + group = {reader.word(), reader.word(), reader.word(), reader.word(), reader.word()}; + } + input.successor_offsets = reader.words(std::uint64_t(input.blocks) + 1); + input.successors = reader.words(input.edges); + input.uses = reader.words(input.cells); + input.defs = reader.words(input.cells); + input.phi_out = reader.words(input.cells); + validate(input); + return input; +} + +static std::string json_string(const std::string& value) { + std::ostringstream result; + result << '"'; + for (const unsigned char ch : value) { + switch (ch) { + case '"': result << "\\\""; break; + case '\\': result << "\\\\"; break; + case '\n': result << "\\n"; break; + case '\r': result << "\\r"; break; + case '\t': result << "\\t"; break; + default: + if (ch < 0x20) { + result << "\\u" << std::hex << std::setw(4) << std::setfill('0') + << static_cast(ch) << std::dec; + } else result << ch; + } + } + return result.str() + '"'; +} + +#ifndef GLC_FORMAT_CHECK_ONLY + +class CudaError : public std::runtime_error { +public: + const cudaError_t code; + CudaError(cudaError_t code, const std::string& operation) + : std::runtime_error(operation + ": " + cudaGetErrorString(code)), code(code) {} +}; + +static void cuda_check(cudaError_t code, const char* operation) { + if (code != cudaSuccess) throw CudaError(code, operation); +} + +template class DeviceBuffer { +public: + T* pointer = nullptr; + const std::size_t count; + explicit DeviceBuffer(std::size_t count) : count(count) { + if (count) cuda_check(cudaMalloc(reinterpret_cast(&pointer), count * sizeof(T)), "cudaMalloc"); + } + explicit DeviceBuffer(const std::vector& values) : DeviceBuffer(values.size()) { + if (count) cuda_check(cudaMemcpy(pointer, values.data(), count * sizeof(T), cudaMemcpyHostToDevice), "cudaMemcpy H2D"); + } + ~DeviceBuffer() { if (pointer) cudaFree(pointer); } + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + void upload(const std::vector& source) const { + if (source.size() != count) throw std::runtime_error("upload size mismatch"); + if (count) cuda_check(cudaMemcpy(pointer, source.data(), count * sizeof(T), cudaMemcpyHostToDevice), "cudaMemcpy H2D"); + } + void download(std::vector& destination) const { + if (destination.size() != count) throw std::runtime_error("download size mismatch"); + if (count) cuda_check(cudaMemcpy(destination.data(), pointer, count * sizeof(T), cudaMemcpyDeviceToHost), "cudaMemcpy D2H"); + } +}; + +class Event { +public: + cudaEvent_t value = nullptr; + Event() { cuda_check(cudaEventCreate(&value), "cudaEventCreate"); } + ~Event() { if (value) cudaEventDestroy(value); } + Event(const Event&) = delete; + Event& operator=(const Event&) = delete; +}; + +__global__ void liveness( + const Group* groups, + const std::uint32_t* successor_offsets, + const std::uint32_t* successors, + const std::uint32_t* uses, + const std::uint32_t* defs, + const std::uint32_t* phi_out, + std::uint32_t* result, + std::uint32_t* iterations, + std::uint32_t* converged) { + const Group group = groups[blockIdx.x]; + extern __shared__ std::uint32_t slab[]; + std::uint32_t* previous = slab; + std::uint32_t* next = slab + group.block_count; + __shared__ unsigned int changed; + const unsigned int lane = threadIdx.x; + for (std::uint32_t block = lane; block < group.block_count; block += blockDim.x) { + previous[block] = 0; + } + __syncthreads(); + + bool stable = false; + std::uint32_t sweeps = 0; + // N+1 includes the no-change sweep after propagation along a simple path. + for (std::uint32_t round = 0; round <= group.block_count; ++round) { + if (lane == 0) changed = 0; + __syncthreads(); + for (std::uint32_t block = lane; block < group.block_count; block += blockDim.x) { + const auto global_block = group.block_base + block; + const auto cell = group.row_base + block * group.words + group.word; + std::uint32_t live_out = phi_out[cell]; + for (auto edge = successor_offsets[global_block]; + edge < successor_offsets[global_block + 1]; ++edge) { + live_out |= previous[successors[edge] - group.block_base]; + } + const auto value = uses[cell] | (live_out & ~defs[cell]); + next[block] = value; + if (value != previous[block]) atomicOr(&changed, 1u); + } + __syncthreads(); + sweeps = round + 1; + // changed is block-uniform after the barrier, so every thread takes + // the same exit and every remaining barrier is reached together. + if (changed == 0) { + stable = true; + break; + } + for (std::uint32_t block = lane; block < group.block_count; block += blockDim.x) { + previous[block] = next[block]; + } + __syncthreads(); + } + for (std::uint32_t block = lane; block < group.block_count; block += blockDim.x) { + const auto cell = group.row_base + block * group.words + group.word; + result[cell] = previous[block]; + } + if (lane == 0) { + iterations[blockIdx.x] = sweeps; + converged[blockIdx.x] = stable ? 1u : 0u; + } +} + +static double elapsed_ms(std::chrono::steady_clock::time_point start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +static void write_result(const std::string& path, const std::vector& values) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot open output: " + path); + output.write("GLR1", 4); + auto write_word = [&output](std::uint32_t word) { + const unsigned char bytes[4] = { + static_cast(word), static_cast(word >> 8), + static_cast(word >> 16), static_cast(word >> 24), + }; + output.write(reinterpret_cast(bytes), 4); + }; + write_word(static_cast(values.size())); + for (const auto value : values) write_word(value); + output.close(); + if (!output) throw std::runtime_error("failed writing output: " + path); +} + +static std::string statistics(const std::vector& values) { + auto sorted = values; + std::sort(sorted.begin(), sorted.end()); + std::ostringstream result; + result << std::setprecision(10) << "{\"median_ms\":" << sorted[sorted.size() / 2] + << ",\"min_ms\":" << sorted.front() << ",\"max_ms\":" << sorted.back() + << ",\"samples_ms\":["; + for (std::size_t i = 0; i < values.size(); ++i) { + if (i) result << ','; + result << values[i]; + } + result << "]}"; + return result.str(); +} + +static int run_cuda(int argc, char** argv) { + if (argc < 3 || argc > 4) { + std::cerr << "Usage: cuda-liveness INPUT.bin OUTPUT.bin [repeats]\n"; + return 2; + } + if (std::string(argv[1]) == argv[2]) throw std::runtime_error("output must not overwrite input"); + int repeats = 7; + if (argc == 4) { + std::size_t consumed = 0; + repeats = std::stoi(argv[3], &consumed); + if (consumed != std::string(argv[3]).size() || repeats < 1 || repeats > 10000) { + throw std::runtime_error("repeats must be between 1 and 10000"); + } + } + const auto input = read_input(argv[1]); + const auto initialization_start = std::chrono::steady_clock::now(); + cuda_check(cudaSetDevice(0), "cudaSetDevice(0)"); + cudaDeviceProp properties{}; + cuda_check(cudaGetDeviceProperties(&properties, 0), "cudaGetDeviceProperties"); + constexpr unsigned int threads = 64; + if (properties.maxThreadsPerBlock < threads || input.groups > static_cast(properties.maxGridSize[0])) { + throw std::runtime_error("launch dimensions exceed the device limits"); + } + cudaFuncAttributes attributes{}; + cuda_check(cudaFuncGetAttributes(&attributes, liveness), "cudaFuncGetAttributes"); + const auto dynamic_bytes = std::uint64_t(input.max_blocks) * 2 * sizeof(std::uint32_t); + const auto available_shared = std::max(properties.sharedMemPerBlock, properties.sharedMemPerBlockOptin); + if (dynamic_bytes + attributes.sharedSizeBytes > available_shared || + dynamic_bytes > std::numeric_limits::max()) { + throw std::runtime_error("function exceeds device shared memory; CUDA pass cannot run this input"); + } + if (dynamic_bytes + attributes.sharedSizeBytes > properties.sharedMemPerBlock) { + cuda_check(cudaFuncSetAttribute(liveness, cudaFuncAttributeMaxDynamicSharedMemorySize, + static_cast(dynamic_bytes)), "cudaFuncSetAttribute shared memory"); + } + + DeviceBuffer descriptors(input.descriptors); + DeviceBuffer offsets(input.successor_offsets), successors(input.successors); + DeviceBuffer uses(input.uses), defs(input.defs), phi(input.phi_out); + DeviceBuffer result(input.cells), iterations(input.groups), converged(input.groups); + std::vector host_result(input.cells), host_iterations(input.groups), host_converged(input.groups); + Event begin, end; + const auto initialization_ms = elapsed_ms(initialization_start); + std::uint32_t maximum_sweeps = 0; + + auto run = [&]() { + const auto start = std::chrono::steady_clock::now(); + cuda_check(cudaEventRecord(begin.value), "cudaEventRecord begin"); + liveness<<(dynamic_bytes)>>>( + descriptors.pointer, offsets.pointer, successors.pointer, uses.pointer, + defs.pointer, phi.pointer, result.pointer, iterations.pointer, converged.pointer); + cuda_check(cudaGetLastError(), "liveness launch"); + cuda_check(cudaEventRecord(end.value), "cudaEventRecord end"); + cuda_check(cudaEventSynchronize(end.value), "cudaEventSynchronize"); + float execution_ms = 0; + cuda_check(cudaEventElapsedTime(&execution_ms, begin.value, end.value), "cudaEventElapsedTime"); + converged.download(host_converged); + iterations.download(host_iterations); + result.download(host_result); + for (std::size_t i = 0; i < host_converged.size(); ++i) { + if (host_converged[i] != 1) throw std::runtime_error("CUDA fixed point failed to converge for group " + std::to_string(i)); + maximum_sweeps = std::max(maximum_sweeps, host_iterations[i]); + } + return std::pair{elapsed_ms(start), execution_ms}; + }; + + const auto first = run(); + const auto reference = host_result; + std::vector elapsed, execution; + for (int repeat = 0; repeat < repeats; ++repeat) { + const auto timing = run(); + if (host_result != reference) throw std::runtime_error("CUDA output changed between identical repetitions"); + elapsed.push_back(timing.first); execution.push_back(timing.second); + } + // A compilation produces new input. Time uploading all input arrays as well + // as solving and returning the output, while retaining device allocations. + std::vector input_update_elapsed; + for (int repeat = 0; repeat < repeats; ++repeat) { + const auto start = std::chrono::steady_clock::now(); + descriptors.upload(input.descriptors); + offsets.upload(input.successor_offsets); successors.upload(input.successors); + uses.upload(input.uses); defs.upload(input.defs); phi.upload(input.phi_out); + run(); + input_update_elapsed.push_back(elapsed_ms(start)); + if (host_result != reference) throw std::runtime_error("CUDA output changed after input upload"); + } + write_result(argv[2], host_result); + std::cout << std::setprecision(10) + << "{\"backend\":\"cuda\",\"device_name\":" << json_string(properties.name) + << ",\"device_ordinal\":0,\"threads\":" << threads + << ",\"functions\":" << input.functions << ",\"blocks\":" << input.blocks + << ",\"cells\":" << input.cells << ",\"groups\":" << input.groups + << ",\"max_blocks\":" << input.max_blocks << ",\"shared_memory_bytes\":" << dynamic_bytes + << ",\"repeats\":" << repeats << ",\"max_sweeps\":" << maximum_sweeps + << ",\"converged\":true,\"repeat_outputs_equal\":true" + << ",\"initialization_ms\":" << initialization_ms + << ",\"first_pass_end_to_end_ms\":" << first.first + << ",\"gpu_execution_ms\":" << statistics(execution) + << ",\"warm_pass_end_to_end_ms\":" << statistics(elapsed) + << ",\"warm_input_update_end_to_end_ms\":" << statistics(input_update_elapsed) + << ",\"timing_scope\":\"Warm elapsed includes launch, event wait, convergence checks and device-to-host result copies; excludes initialization, input uploads, file IO and cross-backend verification. Input-update elapsed additionally uploads all input arrays and reuses device allocations.\"}\n"; + return 0; +} + +#endif + +int main(int argc, char** argv) { + try { +#ifdef GLC_FORMAT_CHECK_ONLY + if (argc != 2) { + std::cerr << "Usage: cuda-format-check INPUT.bin\n"; + return 2; + } + const auto input = read_input(argv[1]); + std::cout << "{\"format_validated\":true,\"gpu_executed\":false,\"functions\":" + << input.functions << ",\"blocks\":" << input.blocks + << ",\"cells\":" << input.cells << ",\"groups\":" << input.groups << "}\n"; + return 0; +#else + return run_cuda(argc, argv); +#endif + } +#ifndef GLC_FORMAT_CHECK_ONLY + catch (const CudaError& error) { + std::cerr << "{\"error\":" << json_string(error.what()) + << ",\"cuda_error_code\":" << static_cast(error.code) << "}\n"; + const int code = static_cast(error.code); + return code > 0 && code <= 255 ? code : 1; + } +#endif + catch (const std::exception& error) { + std::cerr << "{\"error\":" << json_string(error.what()) << "}\n"; + return 1; + } +} diff --git a/experiments/gpu-rust-compiler/cuda/validation.Dockerfile b/experiments/gpu-rust-compiler/cuda/validation.Dockerfile new file mode 100644 index 00000000000..e5e10ca6711 --- /dev/null +++ b/experiments/gpu-rust-compiler/cuda/validation.Dockerfile @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# CUDA 13.3.0 / Ubuntu 24.04 development base, also pinned by NemoClaw main at +# 95d6505265ba1f2cb05355f83786353e654de096. Keep its NVIDIA driver constraints. +FROM docker.io/nvidia/cuda@sha256:ef2203909e80b8b976cfc672f7e2ae2b00bc0e25c404ee86d89e10a3802f1c52 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Validation-only distro packages resolve at build time; CI retains their versions. +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates clang-18 g++ python3 cmake ninja-build git curl jq xz-utils zlib1g-dev libzstd-dev libxml2-dev \ + && ln -s /usr/bin/clang-18 /usr/local/bin/clang \ + && rm -rf /var/lib/apt/lists/* + +# Build-only development assets. Deployed adapters use the existing rustc LLVM +# library for fallback; the 126 MB llc executable is not a required compiler asset. +ENV RUSTUP_HOME=/opt/rustup CARGO_HOME=/opt/cargo +ENV PATH=/opt/cargo/bin:/opt/llvm/bin:$PATH +# Match versions.json for the complete NemoClaw corpus build. +RUN curl --fail --location --retry 3 \ + https://github.com/protocolbuffers/protobuf/releases/download/v36.1/protoc-36.1-linux-x86_64.zip \ + --output /tmp/protoc.zip \ + && echo 'c4bc672d9d49214dc8cafdceadf4df92182d6ca8e3ec65a56b2d7de5602669b4 /tmp/protoc.zip' | sha256sum --check - \ + && python3 -m zipfile -e /tmp/protoc.zip /opt/protoc \ + && chmod +x /opt/protoc/bin/protoc \ + && rm /tmp/protoc.zip +ENV PROTOC=/opt/protoc/bin/protoc +RUN curl --fail --location --retry 3 \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz \ + --output /tmp/llvm.tar.xz \ + && echo 'df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384 /tmp/llvm.tar.xz' | sha256sum --check - \ + && mkdir -p /opt/llvm \ + && tar -xJf /tmp/llvm.tar.xz --strip-components=1 -C /opt/llvm \ + && rm /tmp/llvm.tar.xz \ + && curl --fail --location --retry 3 \ + https://static.rust-lang.org/rustup/archive/1.28.2/x86_64-unknown-linux-gnu/rustup-init \ + --output /tmp/rustup-init \ + && echo '20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c /tmp/rustup-init' | sha256sum --check - \ + && chmod +x /tmp/rustup-init \ + && /tmp/rustup-init -y --profile minimal --default-toolchain 1.98.1 --no-modify-path \ + && rm /tmp/rustup-init \ + && rustc +1.98.1 --version --verbose + +# Fetch immutable upstream sources while the image build has network access. +# The qualification container runs without a network and consumes this checkout. +RUN git init /opt/tpde-source \ + && git -C /opt/tpde-source remote add origin https://github.com/tpde2/tpde.git \ + && git -C /opt/tpde-source fetch --depth 1 origin 9779acf4ada3736e779391da1e4b3369dba08024 \ + && git -C /opt/tpde-source checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/tpde-source rev-parse HEAD)" = 9779acf4ada3736e779391da1e4b3369dba08024 \ + && git -C /opt/tpde-source submodule update --init --recursive --depth 1 \ + deps/args deps/spdlog deps/fadec deps/disarm + +WORKDIR /lab diff --git a/experiments/gpu-rust-compiler/fast-backend/CMakeLists.txt b/experiments/gpu-rust-compiler/fast-backend/CMakeLists.txt new file mode 100644 index 00000000000..51c5ecdfeef --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/CMakeLists.txt @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.25) +project(nemoclaw_fast_backend LANGUAGES C CXX) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# This bridge exchanges serialized IR and object bytes, never LLVM object pointers with rustc. +find_package(LLVM 22.1.8 EXACT REQUIRED CONFIG) +# Keep the standalone bridge's required LLVM subset inside its small executable; +# a second complete shared LLVM would count against the compiler asset budget. +set(TPDE_LINK_LLVM_STATIC ON CACHE BOOL "Link LLVM component archives" FORCE) +set(NEMO_TPDE_REVISION "9779acf4ada3736e779391da1e4b3369dba08024") +include(FetchContent) +set(TPDE_INCLUDE_TESTS OFF CACHE BOOL "" FORCE) +set(TPDE_BUILD_DOCS OFF CACHE BOOL "" FORCE) +set(TPDE_ENABLE_ASAN OFF CACHE BOOL "" FORCE) +set(TPDE_ENABLE_LLVM_PLUGIN OFF CACHE BOOL "" FORCE) +set(TPDE_ENABLE_ASSERTIONS ON CACHE BOOL "" FORCE) +FetchContent_Declare(tpde + GIT_REPOSITORY https://github.com/tpde2/tpde.git + GIT_TAG ${NEMO_TPDE_REVISION} + GIT_SUBMODULES deps/args deps/spdlog deps/fadec deps/disarm + GIT_SUBMODULES_RECURSE ON) +FetchContent_MakeAvailable(tpde) + +add_executable(tpde-rust-bridge bridge.cpp leaf_export.cpp) +target_link_libraries(tpde-rust-bridge PRIVATE tpde::tpde_llvm) +target_compile_definitions(tpde-rust-bridge PRIVATE NEMO_TPDE_REVISION="${NEMO_TPDE_REVISION}") +if(NOT LLVM_ENABLE_RTTI) + target_compile_options(tpde-rust-bridge PRIVATE -fno-rtti) +endif() diff --git a/experiments/gpu-rust-compiler/fast-backend/Cargo.lock b/experiments/gpu-rust-compiler/fast-backend/Cargo.lock new file mode 100644 index 00000000000..da5d85ff01d --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/Cargo.lock @@ -0,0 +1,9 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +version = 4 + +[[package]] +name = "nemoclaw-fast-backend" +version = "0.1.0" diff --git a/experiments/gpu-rust-compiler/fast-backend/Cargo.toml b/experiments/gpu-rust-compiler/fast-backend/Cargo.toml new file mode 100644 index 00000000000..0dc1c3db6a1 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "nemoclaw-fast-backend" +version = "0.1.0" +edition = "2021" +publish = false +license = "Apache-2.0" + +[workspace] +resolver = "2" + +[profile.release] +opt-level = 2 diff --git a/experiments/gpu-rust-compiler/fast-backend/GEM1.md b/experiments/gpu-rust-compiler/fast-backend/GEM1.md new file mode 100644 index 00000000000..edf5ab9d33e --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/GEM1.md @@ -0,0 +1,104 @@ + + + +# GEM1 LLVM leaf packet + +`tpde-rust-bridge --export-leaf INPUT OUTPUT` reads and verifies an LLVM text, +bitcode, or bitcode-wrapper input without modifying it. It exports a whole module +only when every defined function satisfies the initial scalar contract. A +rejection returns status 20 without opening the output; invalid LLVM input returns +21/22. The caller must check status and use fresh output paths. + +This is an adapter over LLVM emitted by the Rust frontend. It does not parse Rust +source or bypass Rust's macros, type checking, borrow checking or ABI lowering. +Exporting a packet does not establish GPU execution; the selected device emitter +must produce and execute the corresponding native object separately. + +## Initial eligibility + +- Identified Rust **1.98.1** producer, original x86-64/AArch64 Linux or macOS target, + little-endian layout, input and output at most 2 MiB. +- No global variables, aliases, ifuncs, module assembly or unknown named metadata. + The accepted module flags are PIC/PIE level, unwind tables, frame pointers and + `RtLibUseGOT` (which has no effect on bodies without runtime calls). +- Every definition has external linkage, default visibility/storage and address + space zero; C calling convention; no varargs; zero or one i64 argument and i64 + return; one basic block. Declarations may remain because accepted bodies cannot + call or reference them. +- Constants, wrapping i64 add/sub/mul and return only. `nsw`/`nuw`, undef/poison, + memory operations, calls/intrinsics, control flow and other instructions reject + the entire module. COPY is reserved in the contract but this exporter emits no + copy instruction. +- No personality, COMDAT, explicit section/alignment, prologue/prefix data, + function/instruction/debug metadata or ABI-special parameter/return attributes. + Normal `noundef`/range facts and optimization hints can accompany the accepted + arithmetic without changing its emitted behavior. +- Unknown function attributes reject. Return signing, BTI/PAC/CET/retpoline + requirements, sanitizer/instrumentation and stack-protector attributes are not + represented by this packet and must use a compatible CPU backend. +- Normal Rust `probe-stack=inline-asm` is accepted only with its default 4096-byte + threshold, or an explicit `stack-probe-size` of at least 4096, and a fixed frame + of `align16(value_count * 8) <= 2048`. No stack allocas/calls are permitted. Each + function has at most 256 values. + +`requires_unwind` is true when the function or module requires unwind tables. +The device/object emitter must generate valid fixed-frame unwind information, +including prologue and epilogue transitions for asynchronous unwinding. A +`nounwind` attribute does not cancel an unwind-table requirement. + +Rust wrapping helper methods can leave an unused personality on a pure function +after inlining; that function intentionally falls back. `fixtures/leaf.rs` uses +plain arithmetic under an explicit matched release contract with overflow checks +and debug assertions disabled. Changing quality/hardware selection must not +silently change those semantic flags. + +## Binary layout + +All integers are little endian; strings contain UTF-8 bytes without a terminator. +No host pointers or source AST are serialized. + +| Field | Encoding | +| --- | --- | +| Magic | Four bytes `GEM1` | +| Target triple | u32 byte length, then original UTF-8 bytes | +| Function count | u32 | +| Each function name | u32 byte length, then UTF-8 bytes | +| Parameters / requires_unwind / values / instructions | Four u32 values | +| Each instruction | u32 opcode, u32 result, u32 a, u32 b, i64 immediate bits | + +Value IDs are local to the function, densely assigned in definition order. An +argument uses value zero; constants are materialized before their uses and +deduplicated within the function. All unused operands/immediates are zero. + +| Opcode | Meaning | Fields | +| ---: | --- | --- | +| 0 | ARGUMENT | result 0, a/b/immediate 0; emitted only for one parameter | +| 1 | CONST | result value ID; a/b zero; immediate contains all 64 bits | +| 2 | COPY | Reserved; result and source a; b/immediate zero | +| 3 | ADD | result, a, b; immediate zero | +| 4 | SUB | result, a, b; immediate zero | +| 5 | MUL | result, a, b; immediate zero | +| 6 | RETURN | result `0xffffffff`, a returned value, b/immediate zero | + +The consumer must enforce bounds, unique/ordered definitions, defined operands, +signature/frame limits, one terminal return and the original target. It must +reject unknown versions/opcodes and truncated/trailing bytes. + +## Native contract tests + +The tests use handcrafted LLVM fixtures to check exact packet bytes and strict +rejection. They are ignored by default because they require the compiled LLVM +22.1.8 bridge. Run them explicitly: + +```sh +NEMO_FAST_BACKEND_BRIDGE=/absolute/path/to/tpde-rust-bridge \ + cargo +1.98.1 test --locked --offline \ + --manifest-path experiments/gpu-rust-compiler/fast-backend/Cargo.toml \ + --target-dir experiments/gpu-rust-compiler/fast-backend/.build/controller \ + --test leaf_export -- --ignored +``` + +Capture `fixtures/leaf.rs` through the real frontend with `-Copt-level=1 +-Cpanic=unwind -Coverflow-checks=no -Cdebug-assertions=no`, then export its module +with the original target. Actual native/device execution belongs to the compiler +integration qualification, independently of these protocol tests. diff --git a/experiments/gpu-rust-compiler/fast-backend/NATIVE_PROOF.md b/experiments/gpu-rust-compiler/fast-backend/NATIVE_PROOF.md new file mode 100644 index 00000000000..8c202296324 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/NATIVE_PROOF.md @@ -0,0 +1,144 @@ + + + +# Native adapter verification + +October 10, 2026. The TPDE bridge was built on this Apple ARM64 Mac with two C++ +jobs, using the official LLVM 22.1.8 development archive in a private ignored +directory. No host package or rustup installation was changed. + +The controller's ten behavioral tests and strict Clippy checks passed. New +controller behavior was tested failing before implementation. The native bridge +was separately built and exercised; controller mocks are not its native proof. + +The later explicit LLVM code-generation level extension adds three behavioral +tests and a CLI parser test. Levels 0 through 3 reach direct and fallback LLVM +commands, invalid levels fail before a tool starts, and successful TPDE receipts +do not claim an LLVM quality level. These regressions were observed failing +before implementation. A real controller O1 run over the captured ARM64 semantic +module emitted an 11,552-byte object; the linked driver passed generics, atomics, +caught panics and unwind drops. The O1 receipt identifies target-machine code +generation only; no additional IR optimizer pipeline was introduced. + +The later GEM1 extension exports genuine Rust 1.98.1 scalar leaf modules for +x86-64 Linux and ARM64 Darwin, producing 298-byte packets with original targets +and unwind requirements. Four explicit native contract tests pass: exact packet +bytes, constant-return signatures, ordinary Rust runtime binding flags, and +strict unsupported ABI/protection rejection. Export is CPU preparation for the +device emitter; these checks alone do not establish GPU execution. + +## Observed results + +- `fixtures/semantic.rs` was compiled by Rust 1.98.1 for **x86-64 Linux**, using a + privately unpacked matching Linux standard library. No target triple was + rewritten. The captured IR includes standard-library references, generic + calls, sequentially consistent atomics, `catch_unwind`, and drops during + unwinding. TPDE emitted a structurally validated **13,576-byte ELF object with + fallback disabled**. Its native receipt reports `backend: tpde` and the + observed Rust producer. Linux execution remains a CI qualification step. +- The same Rust source was captured for **Apple ARM64**. TPDE's unsupported + platform selected the explicit LLVM fallback, which emitted a **12,528-byte + Mach-O object**. Linking `fixtures/semantic-driver.rs` against that object and + executing it locally passed generic calls, atomics, caught panics, and unwind + drops. The separately generated Rust/LLVM reference object also passed. +- Real Rust `fixtures/unsupported-asm.rs` for x86-64 Linux was rejected in + required-TPDE mode. No output object was committed. With `--allow-fallback`, + LLVM emitted a validated **928-byte ELF** and the receipt retained the TPDE + rejection. That is fallback coverage, not TPDE coverage. +- A genuine Rust `-Clinker-plugin-lto=yes -Clto=off` bitcode-wrapper `.o` from a + cooperating compiler-seam probe was parsed by the bridge. The controller + preserved its Darwin target and emitted a validated **744-byte Mach-O** through + the documented LLVM fallback. Thus the serialized input seam is usable; this + package alone is not a complete Cargo/rustc wrapper. + +These runs establish adapter execution and the exercised native fallback +semantics. They do not establish full Rust compatibility, full NemoClaw/Omarchy +compilation, executable-size compliance, or GPU acceleration. Receipts identify +the scope as captured LLVM module to object and `gpu_accelerated: false`. + +No performance speedup is claimed. The first LLVM version probe took about 1.6 +seconds during first use of the private executable; subprocess startup, +verification, parsing, teardown, and object checks are included in receipts. +Use repeated, interleaved measurements and an in-process integration before +attributing whole-compiler latency gains. + +## Mach-O fixed-frame unwind verification + +The device emitter's fixed-frame code now retains precise asynchronous DWARF +rows in `__eh_frame`. Mach-O compact records use DWARF mode rather than assuming +the established frame throughout the prologue and epilogue. The records use +PC-relative 64-bit FDE locations with paired SUBTRACTOR/UNSIGNED relocations; +Apple's linker rejected an earlier absolute-pointer encoding. Two-function +objects also require `MH_SUBSECTIONS_VIA_SYMBOLS`: without it, actual native +unwinder lookup failed for the final x86 function because the linked compact +lookup range ended at its first instruction. The flag contract was tested +failing before the fix. + +Both ARM64 and x86-64 two-function objects were linked with the installed Apple +linker. ARM64 functions returned 42 locally; x86-64 functions returned 42 under +Rosetta. LLVM 22.1.8 decoded the linked FDE addresses against their actual symbol +addresses and showed the correct rows at entry, after each frame-establishing +instruction and after frame restoration. System libunwind found both functions +throughout the partial prologue, body and final return instruction, including +the last function in the text section. ARM64 entry lookup used entry+1 because +the local unwinder cursor treats an IP as a return address and searches IP-1. +The two-function unit test covers four relocation entries, their pair ordering, +the subsection flag, retained DWARF data and both compact DWARF encodings. + +Private sources and executables are `unwind-object.rs`, `unwind-lookup.c`, +`unwind-arm-lookup`, and `unwind-x86-lookup` directly under `fast-backend/.build/`. +These checks qualify the fixed-frame metadata emitted for the restricted scalar +ABI. They do not prove arbitrary Rust exception handling or GPU throughput. + +The CUDA emitter received a separate source review of its instruction sizes, +packet ABI, prefix-sum bounds, device ownership and completion fencing. No +evident CUDA compilation defect was found. This Mac has no nvcc or CUDA headers; +actual CUDA compilation and execution remain required runner checks. + +The first Linux hosted run exposed Rust's `.so`-named linker script beside its +actual shared library. The SHA-verified official Rust 1.98.1 Linux archive contains +a 42-byte `INPUT(libLLVM.so.22.1-rust-1.98.1-stable)` script and a +199,557,488-byte ELF. Discovery now checks native binary headers rather than +counting both filename matches. Different native libraries still fail as +ambiguous. Static ELF inspection found all 29 required C-API symbols and confirmed +LLVM 22.1.8 from `LLVMGetVersion` instructions; it did not execute Linux code. +Archive SHA256: `e974f036b28565f37c0f3bd92ddefa809bee16c04f9dcf07b9ed96e05aaaf7c4`. + +## Pinned inputs and tools + +| Item | Identity | +| --- | --- | +| Rust frontend | `1.98.1`, commit `48a229ceaefd4985c50990b14116b6d856af0985`, LLVM `22.1.8` | +| TPDE source | `9779acf4ada3736e779391da1e4b3369dba08024` | +| Official Clang | `22.1.8`, LLVM commit `ca7933e47d3a3451d81e72ac174dcb5aa28b59d1` | +| LLVM archive | `LLVM-22.1.8-macOS-ARM64.tar.xz`, SHA256 `f260f4f7c0d430828a81ae8a3826a1d63fc0963ec2459489308cc23b1f7eab4f` | +| Rust Linux std archive | `rust-std-1.98.1-x86_64-unknown-linux-gnu.tar.xz`, SHA256 `fa3ff450172a16c026944030230c5069947af93c728d9179971d44e5e0cfb561` | +| LLVM configuration header | SHA256 `d30ac6df01f61025a150cb60d30dd80f3929382d0c90579b75639e2201f5c4b6` | +| Initial TPDE bridge before GEM1 extension | SHA256 `45e2d5559af03266548075ab754278ff60bb685631985508126db4445b349b3a` | +| Bridge with GEM1 extension | SHA256 `425be10a5c7cfb87ab6af8a8c647dc5dd6a69d6808422789959ebd0d7b6bdcf0`, 8,086,784 bytes | +| Native `llvm-config` | SHA256 `f1697472090ec136a0b2755bdc5d571517a6f1e6bfd9b2b6f6afe7c66d68dc27` | +| Native `llc` | SHA256 `3d4ea1b357c68e34f70ed531ef8dfcbad7eb767a5cb7b5a486bc3e0d2626e404` | +| Linux `libstd` rlib | SHA256 `bf156342c5f849f27d8f53003131bc7c0e675143389d84dd549b2ebbaccfaa2f` | + +Archive digests matched the publishers' HTTPS metadata. These are provenance +checks, not a claim of binary authentication. TPDE fetched submodules were args +`cc2368ca0d8a962862c96c00fe919e1480050f51`, disarm +`2d13d3f410a52daff1c5d8ef07d623332f372560`, fadec +`3994d89500985491b1a7ccc17827a22058b3de49`, and spdlog +`27cb4c76708608465c413f6d0e6b8d99a4d84302`. + +The private source/build/results are under `fast-backend/.build/`. Native receipts +are `fixtures/semantic-linux-tpde.json`, `fixtures/semantic-linux-llvm.json`, +`fixtures/semantic-darwin-fallback.json`, `fixtures/unsupported-asm-fallback.json`, +and `fixtures/linker-plugin-probe.json`. Captured modules, headers, objects and +executables remain there for inspection. They are not committed binaries. + +The initial raw standalone bridge was 8,065,872 bytes, controller 612,864 bytes, and the +official standalone LLVM fallback tool 126,780,656 bytes. Count required runtime +tools/libraries in any compiler-size comparison. The entire development archive +is build infrastructure, not an appropriate estimate of a final compiler +distribution. Later complete Mac runtime-file accounting uses the compact helper +and reuses Rust's shared LLVM. The compiler plus required adapters/kernels totals +235,129,353 bytes versus the 222,966,640-byte baseline (+5.455%), within the +50% +allowance. It includes the additional Metal libraries actually linked by the +runtime executables. This Mac measurement does not qualify Linux/CUDA asset size. diff --git a/experiments/gpu-rust-compiler/fast-backend/PROVENANCE.md b/experiments/gpu-rust-compiler/fast-backend/PROVENANCE.md new file mode 100644 index 00000000000..6e3d5291948 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/PROVENANCE.md @@ -0,0 +1,27 @@ + + + +# Backend provenance + +The Rust controller and `bridge.cpp` are original experiment code. The bridge uses +TPDE's public `tpde_llvm::LLVMCompiler` API; no upstream implementation is copied. + +- TPDE repository: +- Revision: `9779acf4ada3736e779391da1e4b3369dba08024` +- API declaration: `tpde-llvm/include/tpde-llvm/LLVMCompiler.hpp` +- Upstream CLI inspected for invocation and ownership behavior: + `tpde-llvm/tools/tpde-llc.cpp` at that revision. +- Upstream license: Apache-2.0 WITH LLVM-exception. Fetched source and submodules + retain their notices and license files. Nothing is republished here. +- Required LLVM version: `22.1.8`, matching the experiment's Rust `1.98.1` report. +- Expected Rust producer: `1.98.1`; observed `llvm.ident` is retained in receipts. + +CMake fetches the exact TPDE revision and its recorded submodule revisions. +The bridge reports its compiled TPDE revision and LLVM version; the controller +refuses a mismatch. This is build provenance, not binary signing or authentication. + +The bridge exchanges serialized IR and emitted bytes. It does not pass LLVM +pointers across library builds, replace rustc through a Clang plugin, or preserve +the complete set of rustc code-generation flags automatically. Qualification must +inventory target features, code models, debug data, instrumentation, intrinsics, +unwind behavior, and calling conventions before integrating it into Cargo. diff --git a/experiments/gpu-rust-compiler/fast-backend/README.md b/experiments/gpu-rust-compiler/fast-backend/README.md new file mode 100644 index 00000000000..5ab9c916551 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/README.md @@ -0,0 +1,131 @@ + + + +# Captured Rust LLVM fast-backend experiment + +This package compiles captured LLVM modules into object files through a pinned +TPDE adapter. It is a CPU fast-backend experiment, not GPU acceleration or a +drop-in complete Rust compiler. Its receipt says `gpu_accelerated: false` and +`scope: captured-llvm-module-to-object`. + +The bridge parses and verifies LLVM text or bitcode, preserves the input target, +and emits Linux x86-64/AArch64 ELF. The Rust controller checks backend provenance, +records TPDE failures, and commits only a validated relocatable object. It never +replaces an existing output. Unsupported modules can use LLVM `llc` only +with `--allow-fallback`; the selected backend and reason remain visible. +Use `--backend llvm` for the direct LLVM baseline without attempting TPDE. +`--llvm-codegen-opt-level 0|1|2|3` controls LLVM target-machine code generation; +the standalone default remains 0. The controller passes that level as `-O=N` +without adding an IR optimization pipeline or changing TPDE's code generation. +For integrated builds, choose the level corresponding to the actual rustc +request; Rust's size levels `s` and `z` use target-machine level 2. Receipts +record `llvm_codegen_opt_level` and `llvm_codegen_opt_scope: +llvm-target-machine-codegen-only` when LLVM emits the object, and null for both +fields when TPDE succeeds. The level is not a TPDE quality setting. +The bridge also provides strict [GEM1 leaf export](GEM1.md) for the experimental +GPU native-code emitter. Unsupported whole modules keep the CPU backend route. + +TPDE can mutate its in-memory module. LLVM fallback reparses the original file in +a separate process. Keep captured inputs immutable for the entire request. +Object validation checks format, architecture and structural bounds. It does not +prove machine-code semantics; link and execute differential workloads separately. + +## Build + +The bridge needs an existing LLVM **22.1.8** development installation, matching +Clang, CMake 3.25+, a GNU-compatible C++20 compiler, Python 3.10+ for upstream +encoder generation, and TPDE's fetched dependencies. +CMake fetches the pinned TPDE source. This package does not install system tools. + +```sh +cargo +1.98.1 build --release --locked --offline \ + --manifest-path experiments/gpu-rust-compiler/fast-backend/Cargo.toml \ + --target-dir experiments/gpu-rust-compiler/fast-backend/.build/controller + +cmake -S experiments/gpu-rust-compiler/fast-backend \ + -B experiments/gpu-rust-compiler/fast-backend/.build/native \ + -DCMAKE_BUILD_TYPE=Release -DLLVM_DIR=/path/to/llvm-22.1.8/lib/cmake/llvm \ + -DCMAKE_CXX_COMPILER=/path/to/clang++ -DCMAKE_C_COMPILER=/path/to/clang +cmake --build experiments/gpu-rust-compiler/fast-backend/.build/native \ + --target tpde-rust-bridge --parallel 4 +``` + +Assertions are enabled in the upstream backend for correctness screening. Record +that setting in performance comparisons; changing it requires separate validation. +See [PROVENANCE.md](PROVENANCE.md) for versions and licensing. + +## Compile a captured module + +Capture real Rust IR with the pinned frontend's `--emit=llvm-ir` or +`--emit=llvm-bc`, preserving the source/lockfile/features/target configuration. +Do not time this capture as though it were an integrated fast compiler. + +```sh +experiments/gpu-rust-compiler/fast-backend/.build/controller/release/nemoclaw-fast-backend \ + compile --input captured-rust.ll --output candidate.o \ + --bridge experiments/gpu-rust-compiler/fast-backend/.build/native/tpde-rust-bridge \ + --llc /path/to/llvm-22.1.8/bin/llc --allow-fallback --report receipt.json +``` + +Add `--llvm-codegen-opt-level 1` for an LLVM O1 code-generation comparison; +preserve the captured IR's original optimization settings in the measurement. + +Omit `--allow-fallback` when qualifying TPDE coverage. No host target is inferred +or forced onto input modules. macOS targets route to the LLVM fallback because +current TPDE does not emit Mach-O. Other architectures/platforms are rejected by +the controller. Non-Rust LLVM fixtures are accepted with a null `rustc_ident`; +Rust-produced modules identifying a different compiler version are rejected. + +Receipts include subprocess parsing/probing and object validation in elapsed +time. This screening design intentionally reparses IR and starts processes; it +cannot establish in-process backend latency or complete Cargo acceleration. +Linking, execution, executable size, compile-and-test time, and GPU scaling need +their own measurements. The object backend does not currently enable GPUs. + +## Validate the controller + +```sh +cargo +1.98.1 test --locked --offline \ + --manifest-path experiments/gpu-rust-compiler/fast-backend/Cargo.toml \ + --target-dir experiments/gpu-rust-compiler/fast-backend/.build/controller +``` + +Behavioral tests cover explicit fallback, required-backend failures, provenance, +wrong object architecture, invalid IR, existing-output preservation and literal +argument handling, including all four explicit LLVM code-generation levels for +direct and fallback routes. Native bridge compilation/execution requires LLVM 22.1.8; +controller tests alone do not qualify TPDE or generated Rust machine code. +The [native proof](NATIVE_PROOF.md) records separately observed bridge execution, +matching tool versions, native fallback semantics, and unqualified boundaries. + +## Execute emitted Rust objects + +On a Linux runner with the pinned frontend and bridge, capture the Rust fixture, +compile through TPDE with fallback disabled, and link it to the Rust driver: + +```sh +rustc +1.98.1 --edition=2021 --crate-type=lib -Copt-level=0 \ + --emit=llvm-ir=scalar.ll,obj=reference.o \ + experiments/gpu-rust-compiler/fast-backend/fixtures/scalar.rs + +experiments/gpu-rust-compiler/fast-backend/.build/controller/release/nemoclaw-fast-backend \ + compile --input scalar.ll --output tpde.o \ + --bridge experiments/gpu-rust-compiler/fast-backend/.build/native/tpde-rust-bridge \ + --llc /path/to/llvm-22.1.8/bin/llc --report scalar-tpde.json + +rustc +1.98.1 --edition=2021 \ + experiments/gpu-rust-compiler/fast-backend/fixtures/driver.rs \ + -Clink-arg=tpde.o -o scalar-tpde +./scalar-tpde +``` + +Repeat the driver link with `reference.o` for the LLVM reference. The fixture +checks arithmetic, branches and normal-path Rust drops; it does not qualify +panicking drops, arbitrary intrinsics, async, generics or complete applications. +Capture a full crate's modules and run its appropriate executable workload next. +Keep these generated files in a private build directory, not the source root. + +`fixtures/semantic.rs` and `fixtures/semantic-driver.rs` add standard-library +linkage, generic calls, sequentially consistent atomics, caught panics and drops +while unwinding. Capture and link them the same way with `-Cpanic=unwind`; retain +separate required-TPDE and fallback receipts. A fallback pass is not TPDE coverage. diff --git a/experiments/gpu-rust-compiler/fast-backend/bridge.cpp b/experiments/gpu-rust-compiler/fast-backend/bridge.cpp new file mode 100644 index 00000000000..123ecbf16b3 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/bridge.cpp @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. SPDX-License-Identifier: Apache-2.0 + +// Original adapter code using TPDE's public LLVMCompiler API. See +// PROVENANCE.md. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "leaf_export.hpp" +#include +#include +#include +#include +#include +#include + +static std::string one_line(llvm::StringRef value) { + std::string result; + for (char c : value) { + if (c == '\n') + result += "\\n"; + else if (c == '\r') + result += "\\r"; + else + result += c; + } + return result; +} + +int main(int argc, char **argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + if (mode == "--probe" && argc == 2) { + std::cout << "protocol=1\ntpde_revision=" << NEMO_TPDE_REVISION + << "\nllvm_version=" << LLVM_VERSION_STRING << '\n'; + return 0; + } + if ((mode != "--inspect" || argc != 3) && + (mode != "--compile" || argc != 4) && + (mode != "--export-leaf" || argc != 4)) { + std::cerr << "usage: tpde-rust-bridge --probe | --inspect IR | --compile " + "IR OBJECT | --export-leaf IR PACKET\n"; + return 1; + } + if (mode == "--export-leaf") { + std::error_code error; + const auto size = std::filesystem::file_size(argv[2], error); + if (error || size > 2 * 1024 * 1024) { + std::cerr << "leaf input is unavailable or exceeds 2 MiB\n"; + return 20; + } + } + llvm::LLVMContext context; + llvm::SMDiagnostic diagnostic; + std::unique_ptr module = + llvm::parseIRFile(argv[2], diagnostic, context); + if (!module) { + diagnostic.print(argv[0], llvm::errs()); + return 21; + } + if (llvm::verifyModule(*module, &llvm::errs())) { + std::cerr << "input LLVM module failed verification\n"; + return 22; + } + const auto triple = module->getTargetTriple(); + if (triple.str().empty()) { + std::cerr + << "input must declare its target triple; host inference is disabled\n"; + return 21; + } + if (mode == "--inspect") { + std::cout << "target=" << one_line(triple.str()) << '\n'; + if (const auto *ident = module->getNamedMetadata("llvm.ident")) { + for (const auto *node : ident->operands()) { + for (const auto &operand : node->operands()) { + const auto *text = + llvm::dyn_cast_or_null(operand.get()); + if (text && text->getString().starts_with("rustc version ")) { + std::cout << "rustc_ident=" << one_line(text->getString()) << '\n'; + return 0; + } + } + } + } + return 0; + } + if (mode == "--export-leaf") { + std::vector packet; + std::string error; + if (!exportLeafModule(*module, packet, error)) { + std::cerr << "GPU leaf export rejected: " << error << '\n'; + return 20; + } + std::ofstream output(argv[3], std::ios::binary); + output.write(reinterpret_cast(packet.data()), packet.size()); + output.flush(); + if (!output) { + std::cerr << "writing GEM1 packet failed\n"; + return 24; + } + return 0; + } + // TPDE emits Linux ELF; an Apple target must never receive an ELF object. + if (!triple.isOSLinux()) { + std::cerr << "TPDE target is unsupported: " << triple.str() << '\n'; + return 20; + } + auto compiler = tpde_llvm::LLVMCompiler::create(triple); + if (!compiler) { + std::cerr << "TPDE target is unsupported: " << triple.str() << '\n'; + return 20; + } + std::vector bytes; + if (!compiler->compile_to_elf(*module, bytes)) { + std::cerr << "TPDE rejected an LLVM construct; original input remains " + "available for fallback\n"; + return 23; + } + std::ofstream output(argv[3], std::ios::binary); + output.write(reinterpret_cast(bytes.data()), bytes.size()); + output.flush(); + if (!output) { + std::cerr << "writing emitted object failed\n"; + return 24; + } + return 0; +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/driver.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/driver.rs new file mode 100644 index 00000000000..a92ab331bf5 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/driver.rs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +unsafe extern "C" { + fn experiment_sum(n: u64) -> u64; + fn experiment_mix(x: u64) -> u64; + fn experiment_drop(initial: u64) -> u64; +} + +fn main() { + for n in [0, 1, 3, 1000] { + let expected = (0..n).fold(0u64, u64::wrapping_add); + assert_eq!(unsafe { experiment_sum(n) }, expected); + } + for x in [0, 1, u64::MAX, 0x1234_5678_90ab_cdef] { + let expected = x.rotate_left(13).wrapping_mul(0x9e37_79b9_7f4a_7c15); + assert_eq!(unsafe { experiment_mix(x) }, expected); + assert_eq!(unsafe { experiment_drop(x) }, x.wrapping_add(1)); + } + println!( + "captured Rust object: sums, rotates, wrapping arithmetic and normal-path drops matched" + ); +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/leaf.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/leaf.rs new file mode 100644 index 00000000000..587e3db5a13 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/leaf.rs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![no_std] + +#[unsafe(no_mangle)] +pub extern "C" fn gpu_leaf_math(x: i64) -> i64 { + // Capture under an explicit release contract with overflow checks disabled. + x * 7 + 3 +} + +#[unsafe(no_mangle)] +pub extern "C" fn gpu_leaf_constant() -> i64 { + 42 +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/scalar.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/scalar.rs new file mode 100644 index 00000000000..7f67a77b930 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/scalar.rs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![no_std] + +#[unsafe(no_mangle)] +pub extern "C" fn experiment_sum(n: u64) -> u64 { + let mut sum = 0u64; + let mut i = 0u64; + while i < n { + sum = sum.wrapping_add(i); + i = i.wrapping_add(1); + } + sum +} + +#[unsafe(no_mangle)] +pub extern "C" fn experiment_mix(x: u64) -> u64 { + x.rotate_left(13).wrapping_mul(0x9e37_79b9_7f4a_7c15) +} + +struct Bump<'a>(&'a mut u64); +impl Drop for Bump<'_> { + fn drop(&mut self) { + *self.0 = (*self.0).wrapping_add(1); + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn experiment_drop(initial: u64) -> u64 { + let mut counter = initial; + { + let _guard = Bump(&mut counter); + } + counter +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/semantic-driver.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/semantic-driver.rs new file mode 100644 index 00000000000..83ac264169c --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/semantic-driver.rs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +unsafe extern "C" { + fn experiment_generic_atomic(first: u32, second: u32) -> u64; + fn experiment_unwind_drop() -> u64; +} + +fn main() { + std::panic::set_hook(Box::new(|_| {})); + for (first, second) in [(0, 0), (1, 2), (u32::MAX, u32::MAX)] { + let expected = u64::from(first) + .wrapping_add(u64::from(second)) + .rotate_left(7) + ^ 1; + assert_eq!( + unsafe { experiment_generic_atomic(first, second) }, + expected + ); + } + for _ in 0..3 { + assert_eq!(unsafe { experiment_unwind_drop() }, 101); + } + println!( + "captured Rust object: generic calls, atomics, caught panics and unwind drops matched" + ); +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/semantic.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/semantic.rs new file mode 100644 index 00000000000..48c76c7c4c8 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/semantic.rs @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicU64, Ordering}; + +static NORMAL_DROPS: AtomicU64 = AtomicU64::new(0); +static UNWIND_DROPS: AtomicU64 = AtomicU64::new(0); + +struct Tracked(&'static AtomicU64); +impl Drop for Tracked { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +fn combine>(first: T, second: T) -> u64 { + first.into().wrapping_add(second.into()).rotate_left(7) +} + +#[unsafe(no_mangle)] +pub extern "C" fn experiment_generic_atomic(first: u32, second: u32) -> u64 { + let before = NORMAL_DROPS.load(Ordering::SeqCst); + { + let _guard = Tracked(&NORMAL_DROPS); + } + let after = NORMAL_DROPS.load(Ordering::SeqCst); + combine(first, second) ^ after.wrapping_sub(before) +} + +#[unsafe(no_mangle)] +pub extern "C" fn experiment_unwind_drop() -> u64 { + let before = UNWIND_DROPS.load(Ordering::SeqCst); + let result = std::panic::catch_unwind(|| { + let _guard = Tracked(&UNWIND_DROPS); + panic!("intentional backend unwind fixture"); + }); + let after = UNWIND_DROPS.load(Ordering::SeqCst); + u64::from(result.is_err()) * 100 + after.wrapping_sub(before) +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/unsupported-asm-driver.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/unsupported-asm-driver.rs new file mode 100644 index 00000000000..a95bd784e70 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/unsupported-asm-driver.rs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +unsafe extern "C" { + fn experiment_inline_asm() -> u64; +} + +fn main() { + assert_eq!(unsafe { experiment_inline_asm() }, 42); + println!("captured Rust inline assembly: fallback result matched"); +} diff --git a/experiments/gpu-rust-compiler/fast-backend/fixtures/unsupported-asm.rs b/experiments/gpu-rust-compiler/fast-backend/fixtures/unsupported-asm.rs new file mode 100644 index 00000000000..9f7d5df486b --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/fixtures/unsupported-asm.rs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![no_std] + +// Capture for x86-64 Linux; this exercises TPDE's explicit unsupported-construct route. +#[unsafe(no_mangle)] +pub extern "C" fn experiment_inline_asm() -> u64 { + unsafe { + core::arch::asm!("pause", options(nomem, nostack, preserves_flags)); + } + 42 +} diff --git a/experiments/gpu-rust-compiler/fast-backend/leaf_export.cpp b/experiments/gpu-rust-compiler/fast-backend/leaf_export.cpp new file mode 100644 index 00000000000..2e824833578 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/leaf_export.cpp @@ -0,0 +1,359 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. SPDX-License-Identifier: Apache-2.0 + +#include "leaf_export.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { +constexpr size_t MaxPacket = 2 * 1024 * 1024; +constexpr uint32_t MaxValues = 256; +constexpr uint32_t NoResult = std::numeric_limits::max(); + +struct Instruction { + uint32_t opcode, result, a, b; + uint64_t immediate; +}; +struct Function { + std::string name; + uint32_t parameters, unwind, values; + std::vector instructions; +}; + +bool reject(std::string &error, const std::string &reason) { + error = reason; + return false; +} +bool utf8(llvm::StringRef text) { + const auto *s = reinterpret_cast(text.data()); + size_t i = 0; + while (i < text.size()) { + const unsigned c = s[i++]; + if (c == 0) + return false; + if (c < 0x80) + continue; + unsigned count = 0, value = 0, minimum = 0; + if (c >= 0xc2 && c <= 0xdf) { + count = 1; + value = c & 31; + minimum = 0x80; + } else if (c >= 0xe0 && c <= 0xef) { + count = 2; + value = c & 15; + minimum = 0x800; + } else if (c >= 0xf0 && c <= 0xf4) { + count = 3; + value = c & 7; + minimum = 0x10000; + } else + return false; + if (i + count > text.size()) + return false; + while (count--) { + const unsigned next = s[i++]; + if ((next & 0xc0) != 0x80) + return false; + value = (value << 6) | (next & 63); + } + if (value < minimum || value > 0x10ffff || + (value >= 0xd800 && value <= 0xdfff)) + return false; + } + return true; +} + +void u32(std::vector &out, uint32_t value) { + for (unsigned shift = 0; shift < 32; shift += 8) + out.push_back(value >> shift); +} +void u64(std::vector &out, uint64_t value) { + for (unsigned shift = 0; shift < 64; shift += 8) + out.push_back(value >> shift); +} +void text(std::vector &out, llvm::StringRef value) { + u32(out, value.size()); + out.insert(out.end(), value.bytes_begin(), value.bytes_end()); +} + +bool functionAttributes(const llvm::Function &fn, std::string &error) { + const auto attrs = fn.getAttributes(); + for (const auto attr : attrs.getFnAttrs()) { + if (attr.isStringAttribute()) { + const auto key = attr.getKindAsString(); + const auto value = attr.getValueAsString(); + if (key == "target-cpu" || key == "tune-cpu") + continue; + if (key == "frame-pointer" && + (value == "all" || value == "non-leaf" || value == "none")) + continue; + if (key == "probe-stack" && value == "inline-asm") + continue; + if (key == "stack-probe-size") { + uint64_t threshold = 0; + if (!value.getAsInteger(10, threshold) && threshold >= 4096) + continue; + } + if (key == "target-features") { + bool protection = false; + llvm::SmallVector features; + value.split(features, ','); + for (auto feature : features) { + if (!feature.starts_with("+")) + continue; + feature = feature.drop_front(); + if (feature == "bti" || feature == "pauth" || feature == "paca" || + feature == "pacg" || feature == "shstk" || feature == "ibt" || + feature.starts_with("retpoline")) + protection = true; + } + if (!protection) + continue; + } + return reject(error, "unsupported function attribute " + key.str()); + } + switch (attr.getKindAsEnum()) { + case llvm::Attribute::NoUnwind: + case llvm::Attribute::UWTable: + case llvm::Attribute::WillReturn: + case llvm::Attribute::MustProgress: + case llvm::Attribute::NoFree: + case llvm::Attribute::NoRecurse: + case llvm::Attribute::NoSync: + case llvm::Attribute::Memory: + case llvm::Attribute::NonLazyBind: + case llvm::Attribute::NoInline: + case llvm::Attribute::AlwaysInline: + case llvm::Attribute::InlineHint: + case llvm::Attribute::Cold: + case llvm::Attribute::OptimizeNone: + case llvm::Attribute::OptimizeForSize: + case llvm::Attribute::MinSize: + case llvm::Attribute::Speculatable: + break; + default: + return reject(error, + "unsupported function attribute " + attr.getAsString()); + } + } + auto abiAttrs = [&](llvm::AttributeSet set) { + for (const auto attr : set) { + if (attr.isStringAttribute() || + (attr.getKindAsEnum() != llvm::Attribute::NoUndef && + attr.getKindAsEnum() != llvm::Attribute::Range)) + return false; + } + return true; + }; + if (!abiAttrs(attrs.getRetAttrs())) + return reject(error, "unsupported return ABI attribute"); + for (unsigned i = 0; i < fn.arg_size(); ++i) { + if (!abiAttrs(attrs.getParamAttrs(i))) + return reject(error, "unsupported parameter ABI attribute"); + } + return true; +} + +bool function(const llvm::Function &fn, uint32_t moduleUnwind, Function &out, + std::string &error) { + if (fn.getLinkage() != llvm::GlobalValue::ExternalLinkage || + fn.getVisibility() != llvm::GlobalValue::DefaultVisibility || + fn.getDLLStorageClass() != llvm::GlobalValue::DefaultStorageClass || + fn.getAddressSpace() != 0) + return reject(error, "function linkage/visibility is not representable"); + if (fn.getCallingConv() != llvm::CallingConv::C || fn.isVarArg() || + fn.arg_size() > 1 || !fn.getReturnType()->isIntegerTy(64)) + return reject( + error, + "function must use C ABI with zero/one i64 argument and i64 return"); + if (fn.arg_size() == 1 && !fn.arg_begin()->getType()->isIntegerTy(64)) + return reject(error, "function argument is not i64"); + if (fn.size() != 1 || fn.hasPersonalityFn() || fn.hasComdat() || + fn.hasSection() || fn.getAlign() || fn.hasPrefixData() || + fn.hasPrologueData() || fn.hasMetadata()) + return reject(error, + "function control flow or metadata is not representable"); + if (!utf8(fn.getName()) || fn.getName().empty() || + fn.getName().size() > 65536) + return reject(error, "function name is invalid or too large"); + if (!functionAttributes(fn, error)) + return false; + out.name = fn.getName().str(); + out.parameters = fn.arg_size(); + out.unwind = moduleUnwind || fn.hasFnAttribute(llvm::Attribute::UWTable); + llvm::DenseMap ids; + if (out.parameters) { + ids[&*fn.arg_begin()] = 0; + out.instructions.push_back({0, 0, 0, 0, 0}); + } + auto value = [&](const llvm::Value *v) -> std::optional { + if (auto found = ids.find(v); found != ids.end()) + return found->second; + const auto *constant = llvm::dyn_cast(v); + if (!constant || !constant->getType()->isIntegerTy(64) || + ids.size() >= MaxValues) + return std::nullopt; + const auto id = static_cast(ids.size()); + ids[v] = id; + out.instructions.push_back({1, id, 0, 0, constant->getZExtValue()}); + return id; + }; + bool returned = false; + for (const auto &inst : fn.front()) { + if (inst.hasMetadata() || inst.hasDbgRecords()) + return reject(error, "instruction/debug metadata is not representable"); + if (const auto *ret = llvm::dyn_cast(&inst)) { + const auto result = value(ret->getReturnValue()); + if (!result) + return reject(error, + "return value is unsupported or frame limit exceeded"); + out.instructions.push_back({6, NoResult, *result, 0, 0}); + returned = true; + continue; + } + const auto *binary = llvm::dyn_cast(&inst); + if (!binary || !binary->getType()->isIntegerTy(64)) + return reject(error, "instruction is outside the scalar leaf subset"); + uint32_t opcode = 0; + switch (binary->getOpcode()) { + case llvm::Instruction::Add: + opcode = 3; + break; + case llvm::Instruction::Sub: + opcode = 4; + break; + case llvm::Instruction::Mul: + opcode = 5; + break; + default: + return reject(error, "arithmetic opcode is unsupported"); + } + if (binary->hasNoSignedWrap() || binary->hasNoUnsignedWrap()) + return reject(error, + "poison-generating arithmetic flags are unsupported"); + const auto a = value(binary->getOperand(0)); + const auto b = value(binary->getOperand(1)); + if (!a || !b || ids.size() >= MaxValues) + return reject(error, "operand is unsupported or frame limit exceeded"); + const auto id = static_cast(ids.size()); + ids[binary] = id; + out.instructions.push_back({opcode, id, *a, *b, 0}); + } + if (!returned) + return reject(error, "leaf function has no return"); + out.values = ids.size(); + // The fixed-frame emitter allocates align16(values * 8); this stays below a + // 4096-byte probe threshold. + if (((out.values * 8 + 15) & ~15u) > 2048) + return reject(error, "fixed frame exceeds 2048 bytes"); + return true; +} +} // namespace + +bool exportLeafModule(const llvm::Module &module, std::vector &packet, + std::string &error) { + const auto triple = module.getTargetTriple(); + if ((triple.getArch() != llvm::Triple::x86_64 && + triple.getArch() != llvm::Triple::aarch64) || + (!triple.isOSLinux() && !triple.isMacOSX()) || + !module.getDataLayout().isLittleEndian()) + return reject(error, "unsupported scalar target or byte order"); + if (!module.global_empty() || !module.alias_empty() || + !module.ifunc_empty() || !module.getModuleInlineAsm().empty()) + return reject( + error, "globals, aliases, ifuncs and module assembly are unsupported"); + for (const auto &node : module.named_metadata()) { + if (node.getName() != "llvm.ident" && node.getName() != "llvm.module.flags") + return reject(error, + "module metadata is unsupported: " + node.getName().str()); + } + bool producer = false; + if (const auto *ident = module.getNamedMetadata("llvm.ident")) { + for (const auto *node : ident->operands()) { + for (const auto &operand : node->operands()) { + const auto *name = + llvm::dyn_cast_or_null(operand.get()); + if (name && name->getString().starts_with("rustc version ")) { + if (!name->getString().starts_with("rustc version 1.98.1 ")) + return reject(error, "Rust producer does not match 1.98.1"); + producer = true; + } + } + } + } + if (!producer) + return reject(error, + "scalar export requires an identified Rust 1.98.1 producer"); + uint32_t moduleUnwind = 0; + if (const auto *flags = module.getModuleFlagsMetadata()) { + for (const auto *entry : flags->operands()) { + if (!entry || entry->getNumOperands() != 3) + return reject(error, "invalid module flag"); + const auto *key = + llvm::dyn_cast_or_null(entry->getOperand(1).get()); + const auto *wrapped = llvm::dyn_cast_or_null( + entry->getOperand(2).get()); + const auto *val = + wrapped ? llvm::dyn_cast(wrapped->getValue()) + : nullptr; + if (!key || !val || val->getBitWidth() > 64) + return reject(error, "unsupported module flag"); + const auto name = key->getString(); + // Accepted bodies have no runtime-library calls, so their addressing + // cannot depend on this flag. + if (name == "RtLibUseGOT" && val->getZExtValue() <= 1) + continue; + if (name != "PIC Level" && name != "PIE Level" && name != "uwtable" && + name != "frame-pointer") + return reject(error, "unsupported module flag " + name.str()); + if (val->getZExtValue() > 2) + return reject(error, "unsupported module flag value " + name.str()); + if (name == "uwtable") + moduleUnwind = val->getZExtValue() != 0; + } + } + std::vector functions; + for (const auto &fn : module) { + if (fn.isDeclaration()) + continue; + Function out; + if (!function(fn, moduleUnwind, out, error)) { + error = fn.getName().str() + ": " + error; + return false; + } + functions.push_back(std::move(out)); + if (functions.size() > 65536) + return reject(error, "function count exceeds limit"); + } + if (functions.empty()) + return reject(error, "module has no defined functions"); + std::vector out{'G', 'E', 'M', '1'}; + text(out, triple.str()); + u32(out, functions.size()); + for (const auto &fn : functions) { + text(out, fn.name); + for (uint32_t field : {fn.parameters, fn.unwind, fn.values, + static_cast(fn.instructions.size())}) + u32(out, field); + for (const auto &inst : fn.instructions) { + for (uint32_t field : {inst.opcode, inst.result, inst.a, inst.b}) + u32(out, field); + u64(out, inst.immediate); + } + if (out.size() > MaxPacket) + return reject(error, "GEM1 packet exceeds 2 MiB"); + } + packet = std::move(out); + return true; +} diff --git a/experiments/gpu-rust-compiler/fast-backend/leaf_export.hpp b/experiments/gpu-rust-compiler/fast-backend/leaf_export.hpp new file mode 100644 index 00000000000..55bca007fb8 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/leaf_export.hpp @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. SPDX-License-Identifier: Apache-2.0 + +#pragma once +#include +#include +#include + +namespace llvm { +class Module; +} +bool exportLeafModule(const llvm::Module &module, std::vector &packet, + std::string &error); diff --git a/experiments/gpu-rust-compiler/fast-backend/src/bin/rust-llvm-llc.rs b/experiments/gpu-rust-compiler/fast-backend/src/bin/rust-llvm-llc.rs new file mode 100644 index 00000000000..6d0d72d59fb --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/src/bin/rust-llvm-llc.rs @@ -0,0 +1,650 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Small object emitter using the existing pinned Rust LLVM shared library. +//! Only LLVM's C API crosses the dynamic boundary. Contexts, buffers, modules, +//! layouts, and target machines are created and disposed in that same library. +//! This avoids carrying a second full llc executable in the compiler assets. + +use std::{ + env, + ffi::{CStr, CString, OsString}, + fs, + io::Read, + os::raw::{c_char, c_int, c_uint, c_void}, + path::{Path, PathBuf}, + process::Command, + ptr, + sync::atomic::{AtomicU64, Ordering}, +}; + +const EXPECTED_LLVM: (u32, u32, u32) = (22, 1, 8); +const EXPECTED_RUST: &str = "1.98.1"; +static NEXT: AtomicU64 = AtomicU64::new(0); +type Ref = *mut c_void; + +#[cfg(unix)] +mod dynamic { + use super::*; + use std::os::unix::ffi::OsStrExt; + #[cfg_attr(target_os = "linux", link(name = "dl"))] + extern "C" { + fn dlopen(path: *const c_char, flags: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, name: *const c_char) -> *mut c_void; + fn dlclose(handle: *mut c_void) -> c_int; + fn dlerror() -> *const c_char; + } + pub fn cpath(path: &Path) -> Result { + CString::new(path.as_os_str().as_bytes()).map_err(|_| "path contains a NUL byte".into()) + } + pub struct Library { + handle: Ref, + pub path: PathBuf, + } + impl Library { + pub fn open(path: PathBuf) -> Result { + let path = fs::canonicalize(path).map_err(|e| format!("LLVM library path: {e}"))?; + let name = cpath(&path)?; + // RTLD_NOW resolves dependencies before any function is used. + #[cfg(target_os = "macos")] + let flags = 2 | 4; // NOW | LOCAL + #[cfg(not(target_os = "macos"))] + let flags = 2; // LOCAL is zero on Linux. + let handle = unsafe { dlopen(name.as_ptr(), flags) }; + if handle.is_null() { + return Err(format!("load existing Rust LLVM library: {}", last_error())); + } + Ok(Self { handle, path }) + } + pub unsafe fn symbol(&self, name: &str) -> Result { + if std::mem::size_of::() != std::mem::size_of::() { + return Err("unsupported function-pointer ABI".into()); + } + let name = CString::new(name).map_err(|_| "symbol name contains NUL")?; + dlerror(); + let address = dlsym(self.handle, name.as_ptr()); + if address.is_null() { + return Err(format!( + "required LLVM C API {} unavailable: {}", + name.to_string_lossy(), + last_error() + )); + } + // T is instantiated only with the exact extern-C signatures from + // pinned LLVM22 llvm-c headers; no Rust or LLVM C++ ABI is assumed. + Ok(std::mem::transmute_copy(&address)) + } + } + impl Drop for Library { + fn drop(&mut self) { + unsafe { + dlclose(self.handle); + } + } + } + fn last_error() -> String { + unsafe { + let error = dlerror(); + if error.is_null() { + "dynamic loader returned no diagnostic".into() + } else { + CStr::from_ptr(error).to_string_lossy().into_owned() + } + } + } +} +#[cfg(not(unix))] +compile_error!("rust-llvm-llc currently supports Unix dynamic loaders only"); + +struct Api { + library: dynamic::Library, + version: (u32, u32, u32), + context_create: unsafe extern "C" fn() -> Ref, + context_dispose: unsafe extern "C" fn(Ref), + create_buffer: unsafe extern "C" fn(*const c_char, *mut Ref, *mut *mut c_char) -> c_int, + buffer_dispose: unsafe extern "C" fn(Ref), + parse_ir: unsafe extern "C" fn(Ref, Ref, *mut Ref, *mut *mut c_char) -> c_int, + module_dispose: unsafe extern "C" fn(Ref), + verify: unsafe extern "C" fn(Ref, c_int, *mut *mut c_char) -> c_int, + message_dispose: unsafe extern "C" fn(*mut c_char), + get_target: unsafe extern "C" fn(Ref) -> *const c_char, + get_layout: unsafe extern "C" fn(Ref) -> *const c_char, + set_layout: unsafe extern "C" fn(Ref, *const c_char), + target_from_triple: unsafe extern "C" fn(*const c_char, *mut Ref, *mut *mut c_char) -> c_int, + machine_create: unsafe extern "C" fn( + Ref, + *const c_char, + *const c_char, + *const c_char, + c_int, + c_int, + c_int, + ) -> Ref, + machine_dispose: unsafe extern "C" fn(Ref), + create_layout: unsafe extern "C" fn(Ref) -> Ref, + layout_string: unsafe extern "C" fn(Ref) -> *mut c_char, + layout_dispose: unsafe extern "C" fn(Ref), + emit: unsafe extern "C" fn(Ref, Ref, *const c_char, c_int, *mut *mut c_char) -> c_int, +} +impl Api { + fn load(path: PathBuf) -> Result { + let library = dynamic::Library::open(path)?; + let version = unsafe { + let get: unsafe extern "C" fn(*mut c_uint, *mut c_uint, *mut c_uint) = + library.symbol("LLVMGetVersion")?; + let (mut major, mut minor, mut patch) = (0, 0, 0); + get(&mut major, &mut minor, &mut patch); + (major, minor, patch) + }; + if version != EXPECTED_LLVM { + return Err(format!( + "existing library reports LLVM {}.{}.{}, expected 22.1.8", + version.0, version.1, version.2 + )); + } + // LLVMParseIRInContext2 parses both textual IR and bitcode, and does + // NOT consume its input buffer. Its LLVM22 header explicitly assigns + // disposal to the caller. Older consuming ParseIRInContext is rejected. + unsafe { + Ok(Self { + context_create: library.symbol("LLVMContextCreate")?, + context_dispose: library.symbol("LLVMContextDispose")?, + create_buffer: library.symbol("LLVMCreateMemoryBufferWithContentsOfFile")?, + buffer_dispose: library.symbol("LLVMDisposeMemoryBuffer")?, + parse_ir: library.symbol("LLVMParseIRInContext2")?, + module_dispose: library.symbol("LLVMDisposeModule")?, + verify: library.symbol("LLVMVerifyModule")?, + message_dispose: library.symbol("LLVMDisposeMessage")?, + get_target: library.symbol("LLVMGetTarget")?, + get_layout: library.symbol("LLVMGetDataLayoutStr")?, + set_layout: library.symbol("LLVMSetDataLayout")?, + target_from_triple: library.symbol("LLVMGetTargetFromTriple")?, + machine_create: library.symbol("LLVMCreateTargetMachine")?, + machine_dispose: library.symbol("LLVMDisposeTargetMachine")?, + create_layout: library.symbol("LLVMCreateTargetDataLayout")?, + layout_string: library.symbol("LLVMCopyStringRepOfTargetData")?, + layout_dispose: library.symbol("LLVMDisposeTargetData")?, + emit: library.symbol("LLVMTargetMachineEmitToFile")?, + library, + version, + }) + } + } + unsafe fn message(&self, message: *mut c_char) -> String { + if message.is_null() { + return "LLVM returned no diagnostic".into(); + } + let text = CStr::from_ptr(message).to_string_lossy().into_owned(); + (self.message_dispose)(message); + text + } + fn initialize(&self, triple: &str) -> Result<(), String> { + let family = if triple.starts_with("x86_64-") || triple.starts_with("x86_64h-") { + "X86" + } else if triple.starts_with("aarch64-") || triple.starts_with("arm64-") { + "AArch64" + } else { + return Err(format!( + "unsupported architecture in module target {triple}" + )); + }; + for suffix in [ + "TargetInfo", + "Target", + "TargetMC", + "AsmPrinter", + "AsmParser", + ] { + unsafe { + let initialize: unsafe extern "C" fn() = self + .library + .symbol(&format!("LLVMInitialize{family}{suffix}"))?; + initialize(); + } + } + Ok(()) + } + fn emit_file(&self, input: &Path, output: &Path, codegen_level: c_int) -> Result<(), String> { + if !(0..=3).contains(&codegen_level) { + return Err("code-generation optimization level must be 0..3".into()); + } + let mut owned = Owned { + api: self, + context: ptr::null_mut(), + buffer: ptr::null_mut(), + module: ptr::null_mut(), + machine: ptr::null_mut(), + }; + unsafe { + owned.context = (self.context_create)(); + if owned.context.is_null() { + return Err("LLVM context allocation failed".into()); + } + let input = dynamic::cpath(input)?; + let mut message = ptr::null_mut(); + if (self.create_buffer)(input.as_ptr(), &mut owned.buffer, &mut message) != 0 { + return Err(format!("read LLVM input: {}", self.message(message))); + } + if owned.buffer.is_null() { + return Err("LLVM input buffer allocation failed".into()); + } + if !message.is_null() { + (self.message_dispose)(message); + } + message = ptr::null_mut(); + if (self.parse_ir)(owned.context, owned.buffer, &mut owned.module, &mut message) != 0 { + return Err(format!("parse LLVM input: {}", self.message(message))); + } + if !message.is_null() { + (self.message_dispose)(message); + } + message = ptr::null_mut(); + if owned.module.is_null() { + return Err("LLVM parser returned no module".into()); + } + // ReturnStatusAction=2 avoids aborting on malformed verifier input. + if (self.verify)(owned.module, 2, &mut message) != 0 { + return Err(format!("verify LLVM module: {}", self.message(message))); + } + if !message.is_null() { + (self.message_dispose)(message); + } + message = ptr::null_mut(); + let target = (self.get_target)(owned.module); + if target.is_null() { + return Err("module has no target triple".into()); + } + let triple = CStr::from_ptr(target) + .to_str() + .map_err(|_| "module target is not UTF-8")? + .to_owned(); + if triple.is_empty() { + return Err("module target triple is empty".into()); + } + self.initialize(&triple)?; + let mut registered = ptr::null_mut(); + if (self.target_from_triple)(target, &mut registered, &mut message) != 0 { + return Err(format!("resolve module target: {}", self.message(message))); + } + if !message.is_null() { + (self.message_dispose)(message); + } + message = ptr::null_mut(); + if registered.is_null() { + return Err("LLVM target resolution returned no target".into()); + } + let cpu = CString::new("generic").unwrap(); + let features = CString::new("").unwrap(); + // Exact LLVM22 C enums: None/Less/Default/Aggressive=0/1/2/3, + // PIC=2, Small=3. This selects machine code generation, not an IR + // optimizer pipeline. Function-level CPU, + // feature, unwind and ABI attributes remain in the parsed module. + owned.machine = (self.machine_create)( + registered, + target, + cpu.as_ptr(), + features.as_ptr(), + codegen_level, + 2, + 3, + ); + if owned.machine.is_null() { + return Err("LLVM target machine allocation failed".into()); + } + let layout = (self.get_layout)(owned.module); + if layout.is_null() || CStr::from_ptr(layout).to_bytes().is_empty() { + let data = (self.create_layout)(owned.machine); + if data.is_null() { + return Err("LLVM target data layout allocation failed".into()); + } + let text = (self.layout_string)(data); + if text.is_null() { + (self.layout_dispose)(data); + return Err("LLVM target data layout string unavailable".into()); + } + (self.set_layout)(owned.module, text); + (self.message_dispose)(text); + (self.layout_dispose)(data); + } + let output = dynamic::cpath(output)?; + // ObjectFile=1. The pinned LLVM22 API takes a const filename. + if (self.emit)( + owned.machine, + owned.module, + output.as_ptr(), + 1, + &mut message, + ) != 0 + { + return Err(format!("emit native object: {}", self.message(message))); + } + if !message.is_null() { + (self.message_dispose)(message); + } + } + Ok(()) + } +} +struct Owned<'a> { + api: &'a Api, + context: Ref, + buffer: Ref, + module: Ref, + machine: Ref, +} +impl Drop for Owned<'_> { + fn drop(&mut self) { + unsafe { + // Dispose the target machine before its context, matching rustc's + // documented ordering. Keep the input buffer through module disposal. + if !self.machine.is_null() { + (self.api.machine_dispose)(self.machine); + } + if !self.module.is_null() { + (self.api.module_dispose)(self.module); + } + if !self.buffer.is_null() { + (self.api.buffer_dispose)(self.buffer); + } + if !self.context.is_null() { + (self.api.context_dispose)(self.context); + } + } + } +} + +fn llvm_library_filename(name: &str) -> bool { + name == "libLLVM.dylib" + || (name.starts_with("libLLVM") && (name.ends_with(".so") || name.contains(".so."))) +} + +fn native_library_header(path: &Path) -> Result { + let mut file = fs::File::open(path) + .map_err(|e| format!("read LLVM library candidate {}: {e}", path.display()))?; + let mut magic = [0u8; 4]; + match file.read_exact(&mut magic) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(false), + Err(error) => { + return Err(format!( + "read LLVM library header {}: {error}", + path.display() + )) + } + } + // Official Linux Rust1.98.1 includes a 42-byte INPUT(...) GNU ld script + // whose .so name resembles the real ELF. Scripts are link-time inputs, + // not a second dlopen-able runtime. Keep ELF and Mach-O/fat candidates; + // the loader and exact C-API version checks perform final qualification. + Ok(matches!( + magic, + [0x7f, b'E', b'L', b'F'] + | [0xce, 0xfa, 0xed, 0xfe] + | [0xcf, 0xfa, 0xed, 0xfe] + | [0xfe, 0xed, 0xfa, 0xce] + | [0xfe, 0xed, 0xfa, 0xcf] + | [0xca, 0xfe, 0xba, 0xbe] + | [0xbe, 0xba, 0xfe, 0xca] + | [0xca, 0xfe, 0xba, 0xbf] + | [0xbf, 0xba, 0xfe, 0xca] + )) +} + +fn library_path() -> Result { + if let Some(path) = env::var_os("RUST_LLVM_LIBRARY") { + if path.is_empty() { + return Err("RUST_LLVM_LIBRARY is empty".into()); + } + return Ok(path.into()); + } + let output = Command::new("rustup") + .args(["run", EXPECTED_RUST, "rustc", "--print", "sysroot"]) + .output() + .map_err(|e| format!("find pinned Rust sysroot: {e}"))?; + if !output.status.success() { + return Err(format!( + "find pinned Rust sysroot: {}", + String::from_utf8_lossy(&output.stderr) + )); + } + let text = String::from_utf8(output.stdout).map_err(|_| "pinned sysroot path is not UTF-8")?; + let directory = PathBuf::from(text.trim()).join("lib"); + discover_library(&directory) +} + +fn discover_library(directory: &Path) -> Result { + let mut candidates = vec![]; + let mut non_native = vec![]; + for entry in fs::read_dir(directory).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if llvm_library_filename(&name) { + let path = fs::canonicalize(entry.path()).map_err(|e| e.to_string())?; + if !native_library_header(&path)? { + non_native.push(path); + } else if !candidates.contains(&path) { + candidates.push(path); + } + } + } + if candidates.len() != 1 { + return Err(format!( + "expected one native Rust LLVM shared library in {}; native candidates: {:?}; ignored non-native candidates: {:?}; set RUST_LLVM_LIBRARY explicitly", + directory.display(),candidates,non_native + )); + } + Ok(candidates.remove(0)) +} + +enum Request { + Version, + Emit { + input: PathBuf, + output: PathBuf, + codegen_level: c_int, + }, + Help, +} +fn parse(arguments: Vec) -> Result { + if arguments == [OsString::from("--version")] { + return Ok(Request::Version); + } + if arguments == [OsString::from("--help")] { + return Ok(Request::Help); + } + let mut input = None; + let mut output = None; + let mut index = 0; + let mut separated = false; + let mut codegen_level = 0; + let mut codegen_supplied = false; + while index < arguments.len() { + let argument = &arguments[index]; + if !separated && argument == "--" { + separated = true; + index += 1; + continue; + } + if !separated && argument == "-o" { + if output.is_some() { + return Err("duplicate -o".into()); + } + index += 1; + output = Some(PathBuf::from( + arguments.get(index).ok_or("-o requires a filename")?, + )); + } else if !separated && argument == "-filetype=obj" { + } else if !separated && argument.to_string_lossy().starts_with("-O=") { + if codegen_supplied { + return Err("duplicate code-generation optimization option".into()); + } + codegen_level = match argument.to_str() { + Some("-O=0") => 0, + Some("-O=1") => 1, + Some("-O=2") => 2, + Some("-O=3") => 3, + _ => { + return Err(format!( + "unsupported LLVM emitter option: {}", + argument.to_string_lossy() + )) + } + }; + codegen_supplied = true; + } else if !separated && argument.to_string_lossy().starts_with('-') { + return Err(format!( + "unsupported LLVM emitter option: {}", + argument.to_string_lossy() + )); + } else if input.replace(PathBuf::from(argument)).is_some() { + return Err("exactly one LLVM input is required".into()); + } + index += 1; + } + let input = input.ok_or("LLVM input is required")?; + let output = output.ok_or("-o output is required")?; + if output.exists() { + return Err("output already exists; refusing to replace it".into()); + } + if input == output { + return Err("input and output must differ".into()); + } + Ok(Request::Emit { + input, + output, + codegen_level, + }) +} +struct Stage(PathBuf); +impl Stage { + fn new(output: &Path) -> Result { + let parent = output + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + for _ in 0..32 { + let directory = parent.join(format!( + ".rust-llvm-llc-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match fs::create_dir(&directory) { + Ok(()) => return Ok(Self(directory)), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e.to_string()), + } + } + Err("cannot create owned LLVM emission staging".into()) + } +} +impl Drop for Stage { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn run() -> Result<(), String> { + let request = parse(env::args_os().skip(1).collect())?; + if matches!(request, Request::Help) { + println!("rust-llvm-llc --version | -filetype=obj [-O=0|1|2|3] -o OBJECT -- LLVM_INPUT\nDefault code generation level is 0. Input IR is not reoptimized. Uses the existing pinned Rust1.98.1 LLVM22.1.8 library. Set RUST_LLVM_LIBRARY to its absolute path."); + return Ok(()); + } + let api = Api::load(library_path()?)?; + match request { + Request::Version => { + println!("Rust LLVM C-API object emitter\nLLVM version {}.{}.{}\nExisting Rust LLVM library: {}\nLibrary bytes: {}\nSupported codegen levels: 0,1,2,3 (default 0)\nPolicy: preserve module target and attributes; PIC/Small; machine-code optimization only, no IR optimizer pipeline",api.version.0,api.version.1,api.version.2,api.library.path.display(),fs::metadata(&api.library.path).map_err(|e|e.to_string())?.len()); + } + Request::Emit { + input, + output, + codegen_level, + } => { + let stage = Stage::new(&output)?; + let candidate = stage.0.join("object.o"); + api.emit_file(&input, &candidate, codegen_level)?; + if fs::metadata(&candidate).map_err(|e| e.to_string())?.len() == 0 { + return Err("LLVM emitted an empty object".into()); + } + // create-only publication protects both a prior artifact and a + // racing writer. Failed or partial LLVM output never becomes final. + fs::hard_link(candidate, output) + .map_err(|e| format!("publish native object without replacement: {e}"))?; + } + Request::Help => unreachable!(), + } + Ok(()) +} +fn main() { + if let Err(error) = run() { + eprintln!("rust-llvm-llc: {error}"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod tests { + #[test] + fn rust_linux_llvm_shared_library_names_support_both_suffix_orders() { + for name in [ + "libLLVM.so", + "libLLVM.so.22.1-rust-1.98.1-stable", + "libLLVM-22-rust-1.98.1-stable.so", + "libLLVM.dylib", + ] { + assert!(super::llvm_library_filename(name), "{name}"); + } + for name in [ + "libLLVM.a", + "libLLVMCore.a", + "notLLVM.so", + "libLLVM.dylib.a", + ] { + assert!(!super::llvm_library_filename(name), "{name}"); + } + } + + #[test] + fn linux_gnu_linker_script_is_not_counted_as_a_runtime_shared_library() { + let stage = super::Stage::new(&std::env::temp_dir().join("llvm-audit.o")).unwrap(); + let library = stage.0.join("libLLVM.so.22.1-rust-1.98.1-stable"); + let script = stage.0.join("libLLVM-22-rust-1.98.1-stable.so"); + std::fs::write(&library, b"\x7fELF-binary-fixture").unwrap(); + std::fs::write(&script, b"INPUT(libLLVM.so.22.1-rust-1.98.1-stable)\n").unwrap(); + let selected = super::discover_library(&stage.0).unwrap(); + assert_eq!(selected, std::fs::canonicalize(library).unwrap()); + } + + #[test] + fn distinct_native_candidates_remain_ambiguous_instead_of_selecting_arbitrary_bytes() { + let stage = super::Stage::new(&std::env::temp_dir().join("llvm-ambiguous.o")).unwrap(); + std::fs::write( + stage.0.join("libLLVM.so.22.1-rust-1.98.1-stable"), + b"\x7fELF-first", + ) + .unwrap(); + std::fs::write(stage.0.join("libLLVM-22-other.so"), b"\x7fELF-second").unwrap(); + let error = super::discover_library(&stage.0).unwrap_err(); + assert!(error.contains("native candidates")); + assert!(error.contains("libLLVM-22-other.so")); + } + + #[test] + #[ignore = "requires the hash-verified official Linux Rust1.98.1 archive extraction"] + fn official_linux_archive_resolves_the_runtime_elf_and_ignores_its_linker_script() { + let directory = std::path::PathBuf::from( + std::env::var_os("NEMO_LLVM_AUDIT_DIRECTORY") + .expect("set NEMO_LLVM_AUDIT_DIRECTORY to extracted rustc/lib"), + ); + let real = directory.join("libLLVM.so.22.1-rust-1.98.1-stable"); + let script = directory.join("libLLVM-22-rust-1.98.1-stable.so"); + assert_eq!(std::fs::metadata(&real).unwrap().len(), 199_557_488); + assert_eq!( + std::fs::read(&script).unwrap(), + b"INPUT(libLLVM.so.22.1-rust-1.98.1-stable)\n" + ); + assert_eq!( + super::discover_library(&directory).unwrap(), + std::fs::canonicalize(real).unwrap() + ); + } +} diff --git a/experiments/gpu-rust-compiler/fast-backend/src/lib.rs b/experiments/gpu-rust-compiler/fast-backend/src/lib.rs new file mode 100644 index 00000000000..a7dd8363d0d --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/src/lib.rs @@ -0,0 +1,820 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +pub const TPDE_REVISION: &str = "9779acf4ada3736e779391da1e4b3369dba08024"; +pub const LLVM_VERSION: &str = "22.1.8"; +pub const RUST_VERSION: &str = "1.98.1"; + +#[derive(Clone, Debug)] +pub struct Invocation { + pub program: PathBuf, + pub args: Vec, +} + +#[derive(Clone, Debug)] +pub struct ToolOutput { + pub success: bool, + pub stdout: String, + pub stderr: String, +} + +pub trait Runner { + fn run(&mut self, invocation: &Invocation) -> Result; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Backend { + Tpde, + Llvm, +} + +#[derive(Clone, Debug)] +pub struct CompileRequest { + pub input: PathBuf, + pub output: PathBuf, + pub bridge: PathBuf, + pub llc: PathBuf, + pub allow_fallback: bool, + pub backend: Backend, + pub llvm_codegen_opt_level: u8, +} + +#[derive(Debug)] +pub struct CompileReport { + pub backend: String, + pub target: String, + pub fallback_reason: Option, + pub object_bytes: u64, + pub elapsed_ms: f64, + pub stages: Vec<(String, f64)>, + pub rustc_ident: Option, + pub llvm_codegen_opt_level: Option, +} + +static NEXT_STAGE: AtomicU64 = AtomicU64::new(0); + +struct StageDir(PathBuf); +impl StageDir { + fn new(parent: &Path) -> Result { + for _ in 0..32 { + let p = parent.join(format!( + ".fast-backend-{}-{}", + std::process::id(), + NEXT_STAGE.fetch_add(1, Ordering::Relaxed) + )); + match fs::create_dir(&p) { + Ok(()) => return Ok(Self(p)), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(format!("create staging directory: {e}")), + } + } + Err("could not create a unique staging directory".into()) + } +} +impl Drop for StageDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn path_arg(p: &Path) -> Result { + p.to_str() + .map(str::to_owned) + .ok_or_else(|| "paths must be UTF-8".into()) +} + +fn fields(text: &str) -> Result, String> { + let mut out = BTreeMap::new(); + for line in text.lines() { + let (key, value) = line.split_once('=').ok_or("invalid bridge protocol")?; + if out.insert(key, value).is_some() { + return Err("duplicate bridge protocol field".into()); + } + } + Ok(out) +} + +fn timed_run( + runner: &mut impl Runner, + invocation: Invocation, + name: &str, + stages: &mut Vec<(String, f64)>, +) -> Result { + let start = Instant::now(); + let result = runner.run(&invocation); + stages.push((name.into(), start.elapsed().as_secs_f64() * 1000.0)); + result +} + +fn tool_failure(label: &str, result: ToolOutput) -> String { + format!("{label}: {}", result.stderr.trim()) +} + +pub fn compile(req: &CompileRequest, runner: &mut impl Runner) -> Result { + let start = Instant::now(); + if req.llvm_codegen_opt_level > 3 { + return Err("LLVM codegen optimization level must be 0..=3".into()); + } + if req.output.exists() { + return Err(format!("output already exists: {}", req.output.display())); + } + if !req.input.is_file() { + return Err(format!("input is not a file: {}", req.input.display())); + } + let parent = req + .output + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let stage = StageDir::new(parent)?; + let temporary = stage.0.join("candidate.o"); + let mut stages = vec![]; + let provenance = timed_run( + runner, + Invocation { + program: req.bridge.clone(), + args: vec!["--probe".into()], + }, + "provenance", + &mut stages, + )?; + if !provenance.success { + return Err(tool_failure("backend provenance failed", provenance)); + } + let p = fields(&provenance.stdout)?; + if p.get("protocol") != Some(&"1") + || p.get("tpde_revision") != Some(&TPDE_REVISION) + || p.get("llvm_version") != Some(&LLVM_VERSION) + { + return Err("backend provenance does not match the pinned TPDE/LLVM versions".into()); + } + let inspection = timed_run( + runner, + Invocation { + program: req.bridge.clone(), + args: vec!["--inspect".into(), path_arg(&req.input)?], + }, + "parse-and-verify", + &mut stages, + )?; + if !inspection.success { + return Err(tool_failure("input verification failed", inspection)); + } + let i = fields(&inspection.stdout)?; + let target = i + .get("target") + .ok_or("input has no target triple")? + .to_string(); + let kind = target_kind(&target)?; + let rustc_ident = i + .get("rustc_ident") + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + if let Some(ref ident) = rustc_ident { + if !ident.starts_with(&format!("rustc version {RUST_VERSION} ")) { + return Err(format!( + "Rust producer does not match pinned {RUST_VERSION}: {ident}" + )); + } + } + let mut fallback_reason = None; + let mut backend = "tpde"; + if req.backend == Backend::Tpde && kind.0 == "elf" { + let fast = timed_run( + runner, + Invocation { + program: req.bridge.clone(), + args: vec![ + "--compile".into(), + path_arg(&req.input)?, + path_arg(&temporary)?, + ], + }, + "tpde-codegen", + &mut stages, + ); + match fast { + Ok(result) if result.success => {} + Ok(result) => fallback_reason = Some(tool_failure("TPDE rejected module", result)), + Err(e) => fallback_reason = Some(format!("TPDE invocation failed: {e}")), + } + } else if req.backend == Backend::Tpde { + fallback_reason = Some(format!("TPDE does not support target {target}")); + } + if let Some(ref reason) = fallback_reason { + if !req.allow_fallback { + return Err(reason.clone()); + } + } + if req.backend == Backend::Llvm || fallback_reason.is_some() { + // A failed backend may have written bytes; never validate or commit those bytes. + if temporary.exists() { + fs::remove_file(&temporary).map_err(|e| format!("remove failed candidate: {e}"))?; + } + let version = timed_run( + runner, + Invocation { + program: req.llc.clone(), + args: vec!["--version".into()], + }, + "llvm-provenance", + &mut stages, + )?; + if !version.success + || !version + .stdout + .lines() + .filter_map(|line| line.split_once("LLVM version ")) + .any(|(_, version)| version.split_whitespace().next() == Some(LLVM_VERSION)) + { + return Err(format!( + "LLVM fallback must report LLVM version {LLVM_VERSION}" + )); + } + let result = timed_run( + runner, + Invocation { + program: req.llc.clone(), + args: vec![ + "-filetype=obj".into(), + format!("-O={}", req.llvm_codegen_opt_level), + "-o".into(), + path_arg(&temporary)?, + "--".into(), + path_arg(&req.input)?, + ], + }, + "llvm-codegen", + &mut stages, + )?; + if !result.success { + return Err(tool_failure("LLVM fallback failed", result)); + } + backend = "llvm"; + } + let validation_start = Instant::now(); + let object_bytes = validate_object(&temporary, &target)?; + stages.push(( + "object-validation".into(), + validation_start.elapsed().as_secs_f64() * 1000.0, + )); + // Hard-link publication creates the destination only if absent, preserving a racing writer. + fs::hard_link(&temporary, &req.output) + .map_err(|e| format!("commit object without replacing output: {e}"))?; + Ok(CompileReport { + backend: backend.into(), + target, + fallback_reason, + object_bytes, + elapsed_ms: start.elapsed().as_secs_f64() * 1000.0, + stages, + rustc_ident, + llvm_codegen_opt_level: (backend == "llvm").then_some(req.llvm_codegen_opt_level), + }) +} + +fn target_kind(target: &str) -> Result<(&'static str, u16), String> { + let machine = if target.starts_with("x86_64-") { + 62 + } else if target.starts_with("aarch64-") || target.starts_with("arm64-") { + 183 + } else { + return Err(format!("unsupported object architecture: {target}")); + }; + if target.contains("-linux-") { + Ok(("elf", machine)) + } else if target.contains("-apple-") && (target.contains("darwin") || target.contains("macosx")) + { + Ok(("macho", machine)) + } else { + Err(format!("unsupported object platform: {target}")) + } +} + +fn u16_at(data: &[u8], at: usize) -> u16 { + u16::from_le_bytes(data[at..at + 2].try_into().unwrap()) +} +fn u32_at(data: &[u8], at: usize) -> u32 { + u32::from_le_bytes(data[at..at + 4].try_into().unwrap()) +} +fn u64_at(data: &[u8], at: usize) -> u64 { + u64::from_le_bytes(data[at..at + 8].try_into().unwrap()) +} + +pub fn validate_object(path: &Path, target: &str) -> Result { + let data = fs::read(path).map_err(|e| format!("read emitted object: {e}"))?; + let (format, machine) = target_kind(target)?; + if format == "elf" { + if data.len() < 64 + || &data[..7] != b"\x7fELF\x02\x01\x01" + || u16_at(&data, 16) != 1 + || u32_at(&data, 20) != 1 + { + return Err("backend did not produce a little-endian ELF64 relocatable object".into()); + } + if u16_at(&data, 18) != machine { + return Err("object machine does not match the input target".into()); + } + let table = usize::try_from(u64_at(&data, 40)).map_err(|_| "invalid ELF section table")?; + let count = usize::from(u16_at(&data, 60)); + if u16_at(&data, 52) != 64 + || u16_at(&data, 58) != 64 + || count < 2 + || table < 64 + || table + .checked_add(count * 64) + .filter(|&end| end <= data.len()) + .is_none() + { + return Err("truncated or unsupported ELF section table".into()); + } + for index in 0..count { + let at = table + index * 64; + // SHT_NOBITS occupies memory but no bytes in the file. + if u32_at(&data, at + 4) != 8 { + let off = u64_at(&data, at + 24); + let len = u64_at(&data, at + 32); + if off + .checked_add(len) + .filter(|&end| end <= data.len() as u64) + .is_none() + { + return Err("ELF section extends beyond emitted object".into()); + } + } + } + } else { + let cpu = if machine == 62 { + 0x0100_0007 + } else { + 0x0100_000c + }; + if data.len() < 32 || u32_at(&data, 0) != 0xfeed_facf || u32_at(&data, 12) != 1 { + return Err("backend did not produce a Mach-O 64-bit relocatable object".into()); + } + if u32_at(&data, 4) != cpu { + return Err("object machine does not match the input target".into()); + } + if 32usize + .checked_add(u32_at(&data, 20) as usize) + .filter(|&end| end <= data.len()) + .is_none() + { + return Err("truncated Mach-O load commands".into()); + } + } + Ok(data.len() as u64) +} + +pub struct ProcessRunner { + pub timeout: Duration, +} +fn read_bounded(mut source: impl Read) -> String { + const LIMIT: usize = 256 * 1024; + let mut saved = Vec::new(); + let mut buf = [0; 8192]; + let mut truncated = false; + loop { + match source.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + let keep = n.min(LIMIT.saturating_sub(saved.len())); + saved.extend_from_slice(&buf[..keep]); + truncated |= keep < n; + } + } + } + let mut out = String::from_utf8_lossy(&saved).into_owned(); + if truncated { + out.push_str("\n[tool output truncated]\n"); + } + out +} +impl Runner for ProcessRunner { + fn run(&mut self, invocation: &Invocation) -> Result { + let mut child = Command::new(&invocation.program) + .args(&invocation.args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("start {}: {e}", invocation.program.display()))?; + let stdout = child.stdout.take().unwrap(); + let stderr = child.stderr.take().unwrap(); + let stdout_thread = std::thread::spawn(move || read_bounded(stdout)); + let stderr_thread = std::thread::spawn(move || read_bounded(stderr)); + let start = Instant::now(); + let result = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status.success()), + Ok(None) if start.elapsed() >= self.timeout => { + let _ = child.kill(); + let _ = child.wait(); + break Err(format!("tool exceeded {} seconds", self.timeout.as_secs())); + } + Ok(None) => std::thread::sleep(Duration::from_millis(10)), + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + break Err(format!("wait for tool: {e}")); + } + } + }; + let stdout = stdout_thread + .join() + .unwrap_or_else(|_| "stdout reader failed".into()); + let stderr = stderr_thread + .join() + .unwrap_or_else(|_| "stderr reader failed".into()); + Ok(ToolOutput { + success: result?, + stdout, + stderr, + }) + } +} + +pub fn json_string(value: &str) -> String { + let mut out = String::from("\""); + for c in value.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c < ' ' => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} +impl CompileReport { + pub fn to_json(&self) -> String { + let stages = self + .stages + .iter() + .map(|(name, ms)| format!("{{\"name\":{},\"elapsed_ms\":{ms:.6}}}", json_string(name))) + .collect::>() + .join(","); + let llvm_level = self + .llvm_codegen_opt_level + .map(|level| level.to_string()) + .unwrap_or_else(|| "null".into()); + let llvm_scope = self + .llvm_codegen_opt_level + .map(|_| json_string("llvm-target-machine-codegen-only")) + .unwrap_or_else(|| "null".into()); + format!("{{\"schema\":1,\"scope\":\"captured-llvm-module-to-object\",\"gpu_accelerated\":false,\"backend\":{},\"target\":{},\"tpde_revision\":{},\"llvm_version\":{},\"llvm_codegen_opt_level\":{},\"llvm_codegen_opt_scope\":{},\"expected_rust_version\":{},\"rustc_ident\":{},\"fallback_reason\":{},\"object_bytes\":{},\"elapsed_ms\":{:.6},\"stages\":[{}]}}\n", json_string(&self.backend), json_string(&self.target), json_string(TPDE_REVISION), json_string(LLVM_VERSION), llvm_level, llvm_scope, json_string(RUST_VERSION), self.rustc_ident.as_deref().map(json_string).unwrap_or_else(|| "null".into()), self.fallback_reason.as_deref().map(json_string).unwrap_or_else(|| "null".into()), self.object_bytes, self.elapsed_ms, stages) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::VecDeque; + use std::fs; + use std::sync::atomic::{AtomicU64, Ordering}; + + static ID: AtomicU64 = AtomicU64::new(0); + + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + let p = std::env::temp_dir().join(format!( + "fast-backend-test-{}-{}", + std::process::id(), + ID.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&p).unwrap(); + Self(p) + } + fn request(&self, allow_fallback: bool) -> CompileRequest { + let input = self.0.join("input with spaces;literal.ll"); + fs::write(&input, "target triple = \"x86_64-unknown-linux-gnu\"\n").unwrap(); + CompileRequest { + input, + output: self.0.join("output with spaces;literal.o"), + bridge: PathBuf::from("bridge with spaces"), + llc: PathBuf::from("llc with spaces"), + allow_fallback, + backend: Backend::Tpde, + llvm_codegen_opt_level: 0, + } + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn elf(machine: u16) -> Vec { + let mut data = vec![0; 200]; + data[..7].copy_from_slice(b"\x7fELF\x02\x01\x01"); + data[16..18].copy_from_slice(&1u16.to_le_bytes()); + data[18..20].copy_from_slice(&machine.to_le_bytes()); + data[20..24].copy_from_slice(&1u32.to_le_bytes()); + data[40..48].copy_from_slice(&72u64.to_le_bytes()); + data[52..54].copy_from_slice(&64u16.to_le_bytes()); + data[58..60].copy_from_slice(&64u16.to_le_bytes()); + data[60..62].copy_from_slice(&2u16.to_le_bytes()); + data[64] = 0xc3; + data[140..144].copy_from_slice(&1u32.to_le_bytes()); + data[144..152].copy_from_slice(&6u64.to_le_bytes()); + data[160..168].copy_from_slice(&64u64.to_le_bytes()); + data[168..176].copy_from_slice(&1u64.to_le_bytes()); + data + } + + fn ok(stdout: &str) -> ToolOutput { + ToolOutput { + success: true, + stdout: stdout.into(), + stderr: String::new(), + } + } + fn provenance() -> ToolOutput { + ok(&format!( + "protocol=1\ntpde_revision={TPDE_REVISION}\nllvm_version={LLVM_VERSION}\n" + )) + } + fn inspected(target: &str) -> ToolOutput { + ok(&format!("target={target}\n")) + } + + struct FakeRunner { + replies: VecDeque, + calls: Vec, + object: Vec, + } + impl Runner for FakeRunner { + fn run(&mut self, invocation: &Invocation) -> Result { + self.calls.push(invocation.clone()); + let reply = self.replies.pop_front().expect("unexpected invocation"); + if invocation.args.first().map(String::as_str) == Some("--compile") { + fs::write(&invocation.args[2], &self.object).unwrap(); + } else if invocation.args.first().map(String::as_str) == Some("-filetype=obj") { + let i = invocation.args.iter().position(|a| a == "-o").unwrap(); + fs::write(&invocation.args[i + 1], &self.object).unwrap(); + } + Ok(reply) + } + } + fn runner(replies: Vec) -> FakeRunner { + FakeRunner { + replies: replies.into(), + calls: vec![], + object: elf(62), + } + } + + #[test] + fn verified_fast_object_is_committed_and_paths_are_separate_arguments() { + let f = Fixture::new(); + let req = f.request(false); + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + ok(""), + ]); + let report = compile(&req, &mut r).unwrap(); + assert_eq!(report.backend, "tpde"); + assert!(report.fallback_reason.is_none()); + assert_eq!(fs::read(&req.output).unwrap(), elf(62)); + assert_eq!(r.calls[2].args[1], req.input.to_str().unwrap()); + assert_eq!(r.calls[2].program, req.bridge); + } + + #[test] + fn unsupported_fast_construct_uses_llvm_only_when_fallback_was_selected() { + let f = Fixture::new(); + let req = f.request(true); + let failure = ToolOutput { + success: false, + stdout: String::new(), + stderr: "unsupported intrinsic".into(), + }; + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + failure, + ok("LLVM version 22.1.8\n"), + ok(""), + ]); + let report = compile(&req, &mut r).unwrap(); + assert_eq!(report.backend, "llvm"); + assert!(report + .fallback_reason + .unwrap() + .contains("unsupported intrinsic")); + assert_eq!(r.calls[4].args.last().unwrap(), req.input.to_str().unwrap()); + } + + #[test] + fn required_fast_backend_never_silently_falls_back() { + let f = Fixture::new(); + let req = f.request(false); + let failure = ToolOutput { + success: false, + stdout: String::new(), + stderr: "unsupported intrinsic".into(), + }; + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + failure, + ]); + assert!(compile(&req, &mut r) + .unwrap_err() + .contains("unsupported intrinsic")); + assert_eq!(r.calls.len(), 3); + assert!(!req.output.exists()); + } + + #[test] + fn wrong_backend_provenance_is_rejected_before_compilation() { + let f = Fixture::new(); + let req = f.request(true); + let mut r = runner(vec![ok( + "protocol=1\ntpde_revision=wrong\nllvm_version=22.1.8\n", + )]); + assert!(compile(&req, &mut r).unwrap_err().contains("provenance")); + assert_eq!(r.calls.len(), 1); + assert!(!req.output.exists()); + } + + #[test] + fn wrong_object_machine_is_rejected_without_committing_output() { + let f = Fixture::new(); + let req = f.request(false); + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + ok(""), + ]); + r.object = elf(183); + assert!(compile(&req, &mut r).unwrap_err().contains("machine")); + assert!(!req.output.exists()); + } + + #[test] + fn truncated_object_and_existing_output_are_rejected() { + let f = Fixture::new(); + let req = f.request(false); + fs::write(&req.output, "existing").unwrap(); + let mut r = runner(vec![]); + assert!(compile(&req, &mut r).unwrap_err().contains("exists")); + assert_eq!(fs::read(&req.output).unwrap(), b"existing"); + fs::remove_file(&req.output).unwrap(); + fs::write(&req.output, b"\x7fELF").unwrap(); + assert!(validate_object(&req.output, "x86_64-unknown-linux-gnu").is_err()); + } + + #[test] + fn invalid_ir_inspection_never_triggers_a_backend_fallback() { + let f = Fixture::new(); + let req = f.request(true); + let mut r = runner(vec![ + provenance(), + ToolOutput { + success: false, + stdout: String::new(), + stderr: "invalid IR".into(), + }, + ]); + assert!(compile(&req, &mut r).unwrap_err().contains("invalid IR")); + assert_eq!(r.calls.len(), 2); + assert!(!req.output.exists()); + } + + #[test] + fn matching_distribution_llvm_version_is_accepted_for_fallback() { + let f = Fixture::new(); + let req = f.request(true); + let failure = ToolOutput { + success: false, + stdout: String::new(), + stderr: "unsupported intrinsic".into(), + }; + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + failure, + ok("Debian LLVM version 22.1.8\n Optimized build.\n"), + ok(""), + ]); + assert_eq!(compile(&req, &mut r).unwrap().backend, "llvm"); + } + + #[test] + fn an_explicit_llvm_baseline_never_attempts_tpde_or_reports_a_fallback() { + let f = Fixture::new(); + let mut req = f.request(false); + req.backend = Backend::Llvm; + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + ok("LLVM version 22.1.8\n"), + ok(""), + ]); + let report = compile(&req, &mut r).unwrap(); + assert_eq!(report.backend, "llvm"); + assert!(report.fallback_reason.is_none()); + assert_eq!(r.calls[2].args, vec!["--version"]); + } + + #[test] + fn selected_llvm_codegen_level_reaches_direct_and_fallback_commands() { + for level in [3, 2, 1, 0] { + for backend in [Backend::Llvm, Backend::Tpde] { + let f = Fixture::new(); + let mut req = f.request(true); + req.backend = backend; + req.llvm_codegen_opt_level = level; + let mut replies = vec![provenance(), inspected("x86_64-unknown-linux-gnu")]; + if backend == Backend::Tpde { + replies.push(ToolOutput { + success: false, + stdout: String::new(), + stderr: "unsupported intrinsic".into(), + }); + } + replies.extend([ok("LLVM version 22.1.8\n"), ok("")]); + let mut r = runner(replies); + let report = compile(&req, &mut r).unwrap(); + let command = r.calls.last().unwrap(); + assert_eq!(command.program, req.llc); + assert_eq!(command.args[1], format!("-O={level}")); + let json = report.to_json(); + assert!(json.contains(&format!("\"llvm_codegen_opt_level\":{level}"))); + assert!(json + .contains("\"llvm_codegen_opt_scope\":\"llvm-target-machine-codegen-only\"")); + } + } + } + + #[test] + fn llvm_level_does_not_change_successful_tpde_commands_or_quality_report() { + let f = Fixture::new(); + let mut req = f.request(false); + req.llvm_codegen_opt_level = 3; + let mut r = runner(vec![ + provenance(), + inspected("x86_64-unknown-linux-gnu"), + ok(""), + ]); + let report = compile(&req, &mut r).unwrap(); + assert_eq!(r.calls.len(), 3); + assert_eq!(r.calls[2].args[0], "--compile"); + assert_eq!(r.calls[2].args.len(), 3); + let json = report.to_json(); + assert!(json.contains("\"llvm_codegen_opt_level\":null")); + assert!(json.contains("\"llvm_codegen_opt_scope\":null")); + } + + #[test] + fn invalid_llvm_codegen_level_is_rejected_before_any_tool_runs() { + let f = Fixture::new(); + let mut req = f.request(true); + req.llvm_codegen_opt_level = 4; + let mut r = runner(vec![]); + assert!(compile(&req, &mut r) + .unwrap_err() + .contains("LLVM codegen optimization level must be 0..=3")); + assert!(r.calls.is_empty()); + assert!(!req.output.exists()); + } + + #[cfg(unix)] + #[test] + fn real_process_runner_preserves_literal_arguments_and_enforces_timeout() { + let mut r = ProcessRunner { + timeout: Duration::from_secs(1), + }; + let value = "spaces ; $() ' \" \\ newline\n"; + let result = r + .run(&Invocation { + program: PathBuf::from("/usr/bin/printf"), + args: vec!["%s".into(), value.into()], + }) + .unwrap(); + assert!(result.success); + assert_eq!(result.stdout, value); + r.timeout = Duration::from_millis(20); + let result = r.run(&Invocation { + program: PathBuf::from("/bin/sleep"), + args: vec!["10".into()], + }); + assert!(result.unwrap_err().contains("exceeded")); + } +} diff --git a/experiments/gpu-rust-compiler/fast-backend/src/main.rs b/experiments/gpu-rust-compiler/fast-backend/src/main.rs new file mode 100644 index 00000000000..b1f3fcc7fb4 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/src/main.rs @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemoclaw_fast_backend::{compile, Backend, CompileRequest, ProcessRunner}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +fn main() { + if let Err(error) = run() { + eprintln!("fast backend: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let mut args = std::env::args().skip(1); + if args.next().as_deref() != Some("compile") { + return Err("usage: nemoclaw-fast-backend compile --input IR --output OBJECT --bridge TPDE_BRIDGE --llc LLVM_LLC [--backend tpde|llvm] [--llvm-codegen-opt-level 0|1|2|3] [--allow-fallback] [--report JSON] [--timeout-seconds 180]".into()); + } + let mut values = BTreeMap::new(); + let mut allow_fallback = false; + while let Some(arg) = args.next() { + if arg == "--allow-fallback" { + if allow_fallback { + return Err("duplicate --allow-fallback".into()); + } + allow_fallback = true; + continue; + } + if ![ + "--input", + "--output", + "--bridge", + "--llc", + "--backend", + "--llvm-codegen-opt-level", + "--report", + "--timeout-seconds", + ] + .contains(&arg.as_str()) + { + return Err(format!("unknown argument: {arg}")); + } + let value = args + .next() + .ok_or_else(|| format!("missing value for {arg}"))?; + if values.insert(arg.clone(), value).is_some() { + return Err(format!("duplicate {arg}")); + } + } + let required = |key: &str| -> Result { + values + .get(key) + .map(PathBuf::from) + .ok_or_else(|| format!("missing {key}")) + }; + let request = CompileRequest { + input: required("--input")?, + output: required("--output")?, + bridge: required("--bridge")?, + llc: required("--llc")?, + allow_fallback, + llvm_codegen_opt_level: parse_llvm_codegen_opt_level( + values.get("--llvm-codegen-opt-level").map(String::as_str), + )?, + backend: match values + .get("--backend") + .map(String::as_str) + .unwrap_or("tpde") + { + "tpde" => Backend::Tpde, + "llvm" => Backend::Llvm, + value => return Err(format!("unknown backend: {value}")), + }, + }; + let timeout = values.get("--timeout-seconds").map_or(Ok(180), |v| { + v.parse::().map_err(|_| "invalid --timeout-seconds") + })?; + if timeout == 0 { + return Err("timeout must be positive".into()); + } + if let Some(path) = values.get("--report") { + if Path::new(path) == request.output || Path::new(path) == request.input { + return Err("report path must differ from input and output".into()); + } + if Path::new(path).exists() { + return Err("report already exists".into()); + } + } + let report = compile( + &request, + &mut ProcessRunner { + timeout: Duration::from_secs(timeout), + }, + )?; + let json = report.to_json(); + if let Some(path) = values.get("--report") { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| format!("create report: {e}"))?; + file.write_all(json.as_bytes()) + .map_err(|e| format!("write report: {e}"))?; + } + print!("{json}"); + Ok(()) +} + +fn parse_llvm_codegen_opt_level(value: Option<&str>) -> Result { + match value.unwrap_or("0") { + "0" => Ok(0), + "1" => Ok(1), + "2" => Ok(2), + "3" => Ok(3), + _ => Err("--llvm-codegen-opt-level must be 0, 1, 2 or 3".into()), + } +} + +#[cfg(test)] +mod tests { + use super::parse_llvm_codegen_opt_level; + + #[test] + fn llvm_codegen_flag_defaults_to_zero_and_rejects_other_values() { + assert_eq!(parse_llvm_codegen_opt_level(None).unwrap(), 0); + for level in 0..=3 { + assert_eq!( + parse_llvm_codegen_opt_level(Some(&level.to_string())).unwrap(), + level + ); + } + for invalid in ["4", "-1", "s", "z", "O2", "", "02"] { + assert!(parse_llvm_codegen_opt_level(Some(invalid)).is_err()); + } + } +} diff --git a/experiments/gpu-rust-compiler/fast-backend/tests/leaf_export.rs b/experiments/gpu-rust-compiler/fast-backend/tests/leaf_export.rs new file mode 100644 index 00000000000..5ab5ebfdf16 --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/tests/leaf_export.rs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::sync::atomic::{AtomicU64, Ordering}; + +static ID: AtomicU64 = AtomicU64::new(0); + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "gem-export-test-{}-{}", + std::process::id(), + ID.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + Self(path) + } + fn export(&self, body: &str) -> (std::process::ExitStatus, String, Option>) { + let input = self.0.join("input.ll"); + let output = self.0.join("output.gem"); + let source = format!("target datalayout = \"e-p:64:64-i64:64-n8:16:32:64-S128\"\ntarget triple = \"x86_64-unknown-linux-gnu\"\n{body}\n!llvm.ident = !{{!0}}\n!0 = !{{!\"rustc version 1.98.1 (48a229cea 2026-09-01)\"}}\n"); + fs::write(&input, source).unwrap(); + let bridge = std::env::var_os("NEMO_FAST_BACKEND_BRIDGE") + .expect("set NEMO_FAST_BACKEND_BRIDGE to the built native bridge"); + let result = Command::new(bridge) + .args(["--export-leaf"]) + .arg(input) + .arg(&output) + .output() + .unwrap(); + ( + result.status, + String::from_utf8_lossy(&result.stderr).into_owned(), + fs::read(output).ok(), + ) + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn u32_field(bytes: &mut Vec, v: u32) { + bytes.extend_from_slice(&v.to_le_bytes()); +} +fn text_field(bytes: &mut Vec, v: &str) { + u32_field(bytes, v.len() as u32); + bytes.extend_from_slice(v.as_bytes()); +} +fn instruction(bytes: &mut Vec, op: u32, result: u32, a: u32, b: u32, imm: i64) { + for v in [op, result, a, b] { + u32_field(bytes, v); + } + bytes.extend_from_slice(&imm.to_le_bytes()); +} + +#[test] +#[ignore = "requires the native LLVM 22.1.8 bridge"] +fn scalar_leaf_packet_preserves_the_target_symbol_unwind_requirement_and_exact_opcode_contract() { + let f = Fixture::new(); + let (status, error, packet) = f.export("define i64 @leaf(i64 %x) uwtable \"probe-stack\"=\"inline-asm\" {\nentry:\n %m = mul i64 %x, 7\n %r = add i64 %m, 3\n ret i64 %r\n}"); + assert!(status.success(), "{error}"); + let mut expected = b"GEM1".to_vec(); + text_field(&mut expected, "x86_64-unknown-linux-gnu"); + u32_field(&mut expected, 1); + text_field(&mut expected, "leaf"); + for v in [1, 1, 5, 6] { + u32_field(&mut expected, v); + } + instruction(&mut expected, 0, 0, 0, 0, 0); + instruction(&mut expected, 1, 1, 0, 0, 7); + instruction(&mut expected, 5, 2, 0, 1, 0); + instruction(&mut expected, 1, 3, 0, 0, 3); + instruction(&mut expected, 3, 4, 2, 3, 0); + instruction(&mut expected, 6, u32::MAX, 4, 0, 0); + assert_eq!(packet.unwrap(), expected); +} + +#[test] +#[ignore = "requires the native LLVM 22.1.8 bridge"] +fn unsafe_or_unrepresentable_module_contracts_are_rejected_without_a_packet() { + let bodies = [ + "define internal i64 @leaf(i64 %x) { ret i64 %x }", + "define hidden i64 @leaf(i64 %x) { ret i64 %x }", + "define weak i64 @leaf(i64 %x) { ret i64 %x }", + "define fastcc i64 @leaf(i64 %x) { ret i64 %x }", + "define i64 @leaf(i64 inreg %x) { ret i64 %x }", + "define i64 @leaf(i64 %x, i64 %y) { ret i64 %x }", + "define i32 @leaf(i64 %x) { ret i32 42 }", + "define i64 @leaf(i64 %x) section \"custom\" { ret i64 %x }", + "declare i32 @personality(...)\ndefine i64 @leaf(i64 %x) personality ptr @personality { ret i64 %x }", + "define i64 @leaf(i64 %x) \"probe-stack\"=\"inline-asm\" \"stack-probe-size\"=\"2048\" { ret i64 %x }", + "define i64 @leaf(i64 %x) { %r = add nsw i64 %x, 1\n ret i64 %r }", + "@state = global i64 0\ndefine i64 @leaf(i64 %x) { ret i64 %x }", + "declare i64 @external(i64)\ndefine i64 @leaf(i64 %x) { %r = call i64 @external(i64 %x)\n ret i64 %r }", + "define i64 @leaf(i64 %x) \"sign-return-address\"=\"all\" { ret i64 %x }", + "define i64 @leaf(i64 %x) \"branch-target-enforcement\" { ret i64 %x }", + "define i64 @leaf(i64 %x) sanitize_address { ret i64 %x }", + ]; + for body in bodies { + let f = Fixture::new(); + let (status, error, packet) = f.export(body); + assert_eq!(status.code(), Some(20), "{body}: {error}"); + assert!(packet.is_none(), "rejected module left a packet: {body}"); + } +} + +#[test] +#[ignore = "requires the native LLVM 22.1.8 bridge"] +fn constant_return_without_an_argument_uses_no_argument_instruction() { + let f = Fixture::new(); + let (status, error, packet) = f.export("define i64 @constant() { ret i64 -1 }"); + assert!(status.success(), "{error}"); + let mut expected = b"GEM1".to_vec(); + text_field(&mut expected, "x86_64-unknown-linux-gnu"); + u32_field(&mut expected, 1); + text_field(&mut expected, "constant"); + for v in [0, 0, 1, 2] { + u32_field(&mut expected, v); + } + instruction(&mut expected, 1, 0, 0, 0, -1); + instruction(&mut expected, 6, u32::MAX, 0, 0, 0); + assert_eq!(packet.unwrap(), expected); +} + +#[test] +#[ignore = "requires the native LLVM 22.1.8 bridge"] +fn normal_runtime_got_module_flag_does_not_block_a_leaf_without_runtime_calls() { + let f = Fixture::new(); + let (status, error, packet) = f.export("define i64 @constant() nonlazybind { ret i64 42 }\n!llvm.module.flags = !{!1}\n!1 = !{i32 1, !\"RtLibUseGOT\", i32 1}"); + assert!(status.success(), "{error}"); + assert!(packet.unwrap().starts_with(b"GEM1")); +} diff --git a/experiments/gpu-rust-compiler/fast-backend/tests/rust_llvm_llc_contract.rs b/experiments/gpu-rust-compiler/fast-backend/tests/rust_llvm_llc_contract.rs new file mode 100644 index 00000000000..423e124ef7d --- /dev/null +++ b/experiments/gpu-rust-compiler/fast-backend/tests/rust_llvm_llc_contract.rs @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); + +fn temporary() -> std::path::PathBuf { + loop { + let path = std::env::temp_dir().join(format!( + "rust-llvm-llc-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + match fs::create_dir(&path) { + Ok(()) => return path, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => panic!("{error}"), + } + } +} + +#[test] +fn version_reports_the_loaded_llvm_instead_of_a_compiled_in_placeholder() { + let output = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")) + .arg("--version") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let text = String::from_utf8(output.stdout).unwrap(); + assert!(text.contains("LLVM version 22.1.8"), "{text}"); + assert!(text.contains("Existing Rust LLVM library:"), "{text}"); + assert!(text.contains("Supported codegen levels: 0,1,2,3"), "{text}"); +} + +#[test] +fn missing_library_and_unsupported_options_fail_without_creating_output() { + let root = temporary(); + let output = root.join("out.o"); + let result = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")) + .env("RUST_LLVM_LIBRARY", root.join("missing.dylib")) + .arg("--version") + .output() + .unwrap(); + assert!(!result.status.success()); + let result = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")) + .args(["-O=4", "-filetype=obj", "-o"]) + .arg(&output) + .args(["--", "input.ll"]) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("unsupported LLVM emitter option")); + assert!(!output.exists()); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn valid_linux_ir_emits_an_elf_object_without_a_second_llvm_distribution() { + let root = temporary(); + let input = root.join("input.ll"); + let output = root.join("out.o"); + fs::write(&input,"target triple = \"x86_64-unknown-linux-gnu\"\ndefine i64 @answer(i64 %x) { %y = add i64 %x, 42\n ret i64 %y }\n").unwrap(); + let result = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")) + .args(["-filetype=obj", "-O=0", "-o"]) + .arg(&output) + .arg("--") + .arg(&input) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(fs::read(&output).unwrap().starts_with(b"\x7fELF")); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn real_rust_bitcode_preserves_generic_calls_atomics_caught_panics_and_unwind_drops() { + check_semantic_codegen("0", "0"); +} + +#[test] +fn codegen_o1_preserves_real_rust_generics_atomics_and_unwind_drops() { + check_semantic_codegen("1", "1"); +} + +#[test] +fn codegen_o2_preserves_real_rust_generics_atomics_and_unwind_drops() { + check_semantic_codegen("1", "2"); +} + +#[test] +fn codegen_o3_preserves_real_rust_generics_atomics_and_unwind_drops() { + check_semantic_codegen("1", "3"); +} + +#[test] +fn absent_codegen_option_matches_explicit_o0_and_conflicting_options_fail() { + let root = temporary(); + let input = root.join("input.ll"); + fs::write(&input,"target triple = \"x86_64-unknown-linux-gnu\"\ndefine i64 @answer(i64 %x) { %y = mul i64 %x, 7\n ret i64 %y }\n").unwrap(); + let emit = |flags: &[&str], name: &str| { + let output = root.join(name); + let result = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")) + .args(flags) + .args(["-filetype=obj", "-o"]) + .arg(&output) + .arg("--") + .arg(&input) + .output() + .unwrap(); + (result, output) + }; + let (default, default_path) = emit(&[], "default.o"); + let (explicit, explicit_path) = emit(&["-O=0"], "explicit.o"); + assert!( + default.status.success(), + "{}", + String::from_utf8_lossy(&default.stderr) + ); + assert!( + explicit.status.success(), + "{}", + String::from_utf8_lossy(&explicit.stderr) + ); + assert_eq!( + fs::read(default_path).unwrap(), + fs::read(explicit_path).unwrap() + ); + let (conflict, conflict_path) = emit(&["-O=1", "-O=2"], "conflict.o"); + assert!(!conflict.status.success()); + assert!(!conflict_path.exists()); + assert!(String::from_utf8_lossy(&conflict.stderr).contains("duplicate code-generation")); + fs::remove_dir_all(root).unwrap(); +} + +fn check_semantic_codegen(frontend_level: &str, codegen_level: &str) { + let root = temporary(); + let input = root.join("semantic.bc"); + let output = root.join("semantic.o"); + let executable = root.join("semantic-driver"); + let fixtures = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures"); + let result = Command::new("rustup") + .args([ + "run", + "1.98.1", + "rustc", + "--edition=2021", + "--crate-type=lib", + "--emit=llvm-bc", + "-Ccodegen-units=1", + "-Cpanic=unwind", + ]) + .arg(format!("-Copt-level={frontend_level}")) + .arg(fixtures.join("semantic.rs")) + .arg("-o") + .arg(&input) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!( + fs::read(&input).unwrap().starts_with(b"BC\xc0\xde") + || fs::read(&input).unwrap().starts_with(b"\xde\xc0\x17\x0b") + ); + let result = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")) + .arg(format!("-O={codegen_level}")) + .args(["-filetype=obj", "-o"]) + .arg(&output) + .arg("--") + .arg(&input) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let result = Command::new("rustup") + .args(["run", "1.98.1", "rustc", "--edition=2021"]) + .arg(fixtures.join("semantic-driver.rs")) + .arg(format!("-Clink-arg={}", output.display())) + .arg("-o") + .arg(&executable) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let result = Command::new(&executable).output().unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert!(String::from_utf8(result.stdout) + .unwrap() + .contains("generic calls, atomics, caught panics and unwind drops matched")); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn malformed_ir_and_existing_outputs_never_publish_partial_or_replace_prior_objects() { + let root = temporary(); + let input = root.join("invalid.ll"); + let output = root.join("out.o"); + fs::write( + &input, + "target triple = \"x86_64-unknown-linux-gnu\"\nnot valid LLVM IR\n", + ) + .unwrap(); + let command = || { + let mut command = Command::new(env!("CARGO_BIN_EXE_rust-llvm-llc")); + command + .args(["-filetype=obj", "-O=0", "-o"]) + .arg(&output) + .arg("--") + .arg(&input); + command + }; + assert!(!command().status().unwrap().success()); + assert!(!output.exists()); + fs::write(&output, b"existing-object").unwrap(); + assert!(!command().status().unwrap().success()); + assert_eq!(fs::read(&output).unwrap(), b"existing-object"); + fs::remove_dir_all(root).unwrap(); +} diff --git a/experiments/gpu-rust-compiler/fixtures/compiler_demo.rs b/experiments/gpu-rust-compiler/fixtures/compiler_demo.rs new file mode 100644 index 00000000000..9ae0e44b943 --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/compiler_demo.rs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// This is ordinary Rust in the explicitly supported compiler subset. +fn mix(value: i64, salt: i64) -> i64 { + let mut result: i64 = value; + if value > salt { + result = value * 3 + salt; + } else { + result = salt * 2 - value; + } + return result; +} + +fn main() -> i64 { + 123 * 456; + let mut index: i64 = 0; + let mut total: i64 = 0; + while index < 40 { + if index % 3 == 0 || index == 7 { + total = total + mix(index, 11); + } else { + total = total + index; + } + index = index + 1; + } + return total; +} diff --git a/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.lock b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.lock new file mode 100644 index 00000000000..9997026c78c --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +version = 4 + +[[package]] +name = "gpu-rust-full-fixture" +version = "0.0.0" +dependencies = [ + "gpu-rust-math-fixture", +] + +[[package]] +name = "gpu-rust-math-fixture" +version = "0.0.0" diff --git a/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.toml b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.toml new file mode 100644 index 00000000000..05a3ce12048 --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/Cargo.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +[workspace] +members = ["math", "driver"] +resolver = "3" diff --git a/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/driver/Cargo.toml b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/driver/Cargo.toml new file mode 100644 index 00000000000..c062a645752 --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/driver/Cargo.toml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "gpu-rust-full-fixture" +version = "0.0.0" +edition = "2024" +publish = false +[dependencies] +gpu-rust-math-fixture = { path = "../math" } diff --git a/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/driver/src/main.rs b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/driver/src/main.rs new file mode 100644 index 00000000000..ff207cefb39 --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/driver/src/main.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +use std::sync::atomic::{AtomicUsize, Ordering}; +static DROPS: AtomicUsize = AtomicUsize::new(0); +struct Guard; +impl Drop for Guard { + fn drop(&mut self) { + DROPS.fetch_add(1, Ordering::SeqCst); + } +} +fn identity(value: T) -> T { + value +} +fn main() { + std::panic::set_hook(Box::new(|_| {})); + let inputs = vec![0i64, 1, -1, i64::MAX, i64::MIN, 0x123456789abcdef0]; + for value in identity(inputs) { + assert_eq!( + gpu_rust_math_fixture::rust_gpu_math(value), + value.wrapping_mul(7).wrapping_add(3) + ); + } + let caught = std::panic::catch_unwind(|| { + let _guard = Guard; + panic!("intentional fixture panic"); + }); + assert!(caught.is_err()); + assert_eq!(DROPS.load(Ordering::SeqCst), 1); + println!("rust-gpu-fixture: arithmetic=verified caught=true drops=1"); +} diff --git a/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/math/Cargo.toml b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/math/Cargo.toml new file mode 100644 index 00000000000..fab3adb4e0c --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/math/Cargo.toml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +[package] +name = "gpu-rust-math-fixture" +version = "0.0.0" +edition = "2024" +publish = false diff --git a/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/math/src/lib.rs b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/math/src/lib.rs new file mode 100644 index 00000000000..b74faec6070 --- /dev/null +++ b/experiments/gpu-rust-compiler/fixtures/full-rust-gpu/math/src/lib.rs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#![no_std] + +// This is a normal Rust dependency compiled by rustc. In the explicitly selected +// release overflow contract, its arithmetic wraps. Debug/checked variants are +// outside the GPU leaf contract and must take the attributed CPU path. +#[unsafe(no_mangle)] +#[inline(never)] +pub extern "C" fn rust_gpu_math(value: i64) -> i64 { + value * 7 + 3 +} diff --git a/experiments/gpu-rust-compiler/native/codegen.metal b/experiments/gpu-rust-compiler/native/codegen.metal new file mode 100644 index 00000000000..20bb9d0b512 --- /dev/null +++ b/experiments/gpu-rust-compiler/native/codegen.metal @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +using namespace metal; + +struct EmitFunction { uint start, count, values, symbol; }; +// The two words preserve the host i64 bit pattern without requiring GPU i64 +// arithmetic. The CPU/shader ABI is 24 bytes with immediate at byte 16. +struct EmitInstruction { uint opcode, result, a, b; uint2 immediate; }; +struct EmitParameters { uint function_count, target, output_capacity, reserved; }; + +kernel void gpuemit_count( + device const EmitFunction *functions [[buffer(0)]], + device const EmitInstruction *instructions [[buffer(1)]], + device uint *lengths [[buffer(2)]], + constant EmitParameters ¶meters [[buffer(3)]], + uint index [[thread_position_in_grid]]) { + if (index >= parameters.function_count) return; + EmitFunction function = functions[index]; + uint length = parameters.target == 0 ? 11 : 12; + for (uint position = 0; position < function.count; ++position) { + uint opcode = instructions[function.start + position].opcode; + if (parameters.target == 0) { + switch (opcode) { + case 0: length += 7; break; + case 1: length += 17; break; + case 2: length += 14; break; + case 3: case 4: length += 24; break; + case 5: length += 25; break; + case 6: length += 9; break; + default: lengths[index] = 0; return; + } + } else { + switch (opcode) { + case 0: length += 4; break; + case 1: length += 20; break; + case 2: length += 8; break; + case 3: case 4: case 5: length += 16; break; + case 6: length += 16; break; + default: lengths[index] = 0; return; + } + } + } + lengths[index] = length; +} + +void emit_byte(device uchar *output, thread uint &position, uchar value) { + output[position++] = value; +} + +void emit_word(device uchar *output, thread uint &position, uint value) { + for (uint byte = 0; byte < 4; ++byte) emit_byte(output, position, uchar(value >> (byte * 8))); +} + +void x86_memory(device uchar *output, thread uint &position, + uchar rex, uchar operation, uchar mode, uint slot) { + emit_byte(output, position, rex); + emit_byte(output, position, operation); + emit_byte(output, position, mode); + emit_word(output, position, uint(-8 * int(slot + 1))); +} + +void x86_emit(device uchar *output, thread uint &position, + EmitFunction function, device const EmitInstruction *instructions) { + emit_byte(output, position, 0x55); + emit_byte(output, position, 0x48); + emit_byte(output, position, 0x89); + emit_byte(output, position, 0xe5); + emit_byte(output, position, 0x48); + emit_byte(output, position, 0x81); + emit_byte(output, position, 0xec); + emit_word(output, position, (function.values * 8 + 15) & ~15u); + for (uint number = 0; number < function.count; ++number) { + EmitInstruction instruction = instructions[function.start + number]; + switch (instruction.opcode) { + case 0: + x86_memory(output, position, 0x48, 0x89, 0xbd, instruction.result); + break; + case 1: + emit_byte(output, position, 0x48); emit_byte(output, position, 0xb8); + emit_word(output, position, instruction.immediate.x); + emit_word(output, position, instruction.immediate.y); + x86_memory(output, position, 0x48, 0x89, 0x85, instruction.result); + break; + case 2: + x86_memory(output, position, 0x48, 0x8b, 0x85, instruction.a); + x86_memory(output, position, 0x48, 0x89, 0x85, instruction.result); + break; + case 3: case 4: case 5: + x86_memory(output, position, 0x48, 0x8b, 0x85, instruction.a); + x86_memory(output, position, 0x4c, 0x8b, 0x95, instruction.b); + if (instruction.opcode == 5) { + emit_byte(output, position, 0x49); emit_byte(output, position, 0x0f); + emit_byte(output, position, 0xaf); emit_byte(output, position, 0xc2); + } else { + emit_byte(output, position, 0x4c); + emit_byte(output, position, instruction.opcode == 3 ? 0x01 : 0x29); + emit_byte(output, position, 0xd0); + } + x86_memory(output, position, 0x48, 0x89, 0x85, instruction.result); + break; + case 6: + x86_memory(output, position, 0x48, 0x8b, 0x85, instruction.a); + emit_byte(output, position, 0xc9); emit_byte(output, position, 0xc3); + break; + } + } +} + +void arm_emit(device uchar *output, thread uint &position, + EmitFunction function, device const EmitInstruction *instructions) { + emit_word(output, position, 0xa9bf7bfd); + emit_word(output, position, 0x910003fd); + emit_word(output, position, 0xd10003ff | (((function.values * 8 + 15) & ~15u) << 10)); + for (uint number = 0; number < function.count; ++number) { + EmitInstruction instruction = instructions[function.start + number]; + switch (instruction.opcode) { + case 0: + emit_word(output, position, 0xf90003e0 | (instruction.result << 10)); + break; + case 1: + for (uint part = 0; part < 4; ++part) { + uint value = part < 2 ? instruction.immediate.x : instruction.immediate.y; + uint chunk = (value >> ((part & 1) * 16)) & 0xffff; + emit_word(output, position, (part == 0 ? 0xd2800009 : 0xf2800009) | + (part << 21) | (chunk << 5)); + } + emit_word(output, position, 0xf90003e9 | (instruction.result << 10)); + break; + case 2: + emit_word(output, position, 0xf94003e9 | (instruction.a << 10)); + emit_word(output, position, 0xf90003e9 | (instruction.result << 10)); + break; + case 3: case 4: case 5: + emit_word(output, position, 0xf94003e9 | (instruction.a << 10)); + emit_word(output, position, 0xf94003ea | (instruction.b << 10)); + emit_word(output, position, instruction.opcode == 3 ? 0x8b0a0129 : + (instruction.opcode == 4 ? 0xcb0a0129 : 0x9b0a7d29)); + emit_word(output, position, 0xf90003e9 | (instruction.result << 10)); + break; + case 6: + emit_word(output, position, 0xf94003e0 | (instruction.a << 10)); + emit_word(output, position, 0x910003bf); + emit_word(output, position, 0xa8c17bfd); + emit_word(output, position, 0xd65f03c0); + break; + } + } +} + +kernel void gpuemit_emit( + device const EmitFunction *functions [[buffer(0)]], + device const EmitInstruction *instructions [[buffer(1)]], + device const uint *offsets [[buffer(2)]], + device const uint *lengths [[buffer(3)]], + device uchar *output [[buffer(4)]], + device uint *statuses [[buffer(5)]], + constant EmitParameters ¶meters [[buffer(6)]], + uint index [[thread_position_in_grid]]) { + if (index >= parameters.function_count) return; + uint offset = offsets[index], length = lengths[index]; + statuses[index] = 0; + if (!length || offset > parameters.output_capacity || length > parameters.output_capacity - offset) return; + uint position = 0; + if (parameters.target == 0) x86_emit(output + offset, position, functions[index], instructions); + else arm_emit(output + offset, position, functions[index], instructions); + statuses[index] = position == length ? 1 : 0; +} diff --git a/experiments/gpu-rust-compiler/native/codegen_bridge.h b/experiments/gpu-rust-compiler/native/codegen_bridge.h new file mode 100644 index 00000000000..21f34e1c68e --- /dev/null +++ b/experiments/gpu-rust-compiler/native/codegen_bridge.h @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +#ifndef GPUEMIT_BRIDGE_H +#define GPUEMIT_BRIDGE_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +typedef struct GPUEmitInstruction { + uint32_t opcode, result, a, b; + int64_t immediate; +} GPUEmitInstruction; +typedef struct GPUEmitFunction {uint32_t start, count, values, symbol;} GPUEmitFunction; +typedef struct GPUEmitInput { + uint32_t function_count, instruction_count, target, reserved; + uint64_t output_capacity; + const GPUEmitFunction *functions; + const GPUEmitInstruction *instructions; +} GPUEmitInput; +typedef struct GPUEmitStats { + double gpu_ms, total_ms; + double host_prefix_ms, count_gpu_ms, emit_gpu_ms; + uint64_t kernel_dispatches, output_bytes; + uint32_t buffer_allocations, reused_pipeline; +} GPUEmitStats; +/* Schema 1: straight-line wrapping i64 leaf C functions, one argument maximum, + * no calls/code relocations. The CPU object writer supplies required fixed-frame + * unwind metadata. target=0 x86-64 SysV; target=1 ARM64. + * submit captures inputs. A context has one outstanding request. finish requires + * output_capacity bytes plus function_count offsets and lengths. A short result + * buffer leaves the request pending. destroy fences pending work before freeing + * storage. Different device-owned contexts may run concurrently. + * Shader path is required for Metal and unused by CUDA. Every failure writes a + * bounded NUL-terminated diagnostic. This is not a complete Rust backend. + */ +void *gpuemit_create(const char *shader_path, uint32_t device_index, char *error, size_t capacity); +int gpuemit_device_id(void *context, char *output, size_t capacity); +int gpuemit_submit(void *context, const GPUEmitInput *input, char *error, size_t capacity); +int gpuemit_finish(void *context, uint8_t *output, size_t output_capacity, + uint32_t *offsets, uint32_t *lengths, size_t function_count, + GPUEmitStats *stats, char *error, size_t capacity); +void gpuemit_destroy(void *context); +#ifdef __cplusplus +} +#endif +#endif diff --git a/experiments/gpu-rust-compiler/native/codegen_cuda.cu b/experiments/gpu-rust-compiler/native/codegen_cuda.cu new file mode 100644 index 00000000000..1ba54ad6158 --- /dev/null +++ b/experiments/gpu-rust-compiler/native/codegen_cuda.cu @@ -0,0 +1,495 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Schema-1 scalar machine-code emission fixture, not a complete Rust backend. +// Each CUDA thread counts/emits one independent straight-line leaf function. +// Counts and the CUB prefix scan remain device-owned. Host validation, capture, +// transfers, completion, and copying the actual written output are timed. + +#include "codegen_bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using Clock = std::chrono::steady_clock; +using U32 = uint32_t; +constexpr size_t kStorageBudget = 512 * 1024 * 1024; +constexpr U32 kThreads = 128; +static_assert(sizeof(GPUEmitInstruction) == 24, "instruction ABI changed"); +static_assert(sizeof(GPUEmitFunction) == 16, "function ABI changed"); +static_assert(sizeof(GPUEmitStats) == 64, "timing ABI changed"); + +void diagnostic(char *output, size_t capacity, const char *message) { + if (output && capacity) std::snprintf(output, capacity, "%s", message); +} +void check(cudaError_t status, const char *operation) { + if (status != cudaSuccess) + throw std::runtime_error(std::string(operation) + ": " + cudaGetErrorString(status)); +} +double elapsed(Clock::time_point start) { + return std::chrono::duration(Clock::now() - start).count(); +} +size_t array_bytes(U32 count, size_t element_size) { + if (count > std::numeric_limits::max() / element_size) + throw std::runtime_error("array size overflows host address space"); + const size_t bytes = size_t(count) * element_size; + if (bytes > kStorageBudget) + throw std::runtime_error("scalar emission array exceeds 512 MiB storage budget"); + return bytes; +} + +// Rust's independently implemented emit_function is the byte oracle. +__device__ U32 instruction_size(U32 opcode, U32 target) { + if (target == 0) { + switch (opcode) { + case 0: return 7; // argument -> stack home + case 1: return 17; // movabs + store + case 2: return 14; // load + store + case 3: case 4: return 24; + case 5: return 25; + case 6: return 9; + } + } else { + switch (opcode) { + case 0: return 4; + case 1: return 20; // MOVZ + three MOVKs + store + case 2: return 8; + case 3: case 4: case 5: return 16; + case 6: return 16; + } + } + return 0; // Host rejects unsupported opcodes before launching. +} + +__global__ void count_functions(const GPUEmitFunction *functions, + const GPUEmitInstruction *instructions, + U32 function_count, U32 target, U32 *lengths) { + const U32 index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= function_count) return; + const GPUEmitFunction function = functions[index]; + U32 size = target == 0 ? 11 : 12; + for (U32 offset = 0; offset < function.count; ++offset) + size += instruction_size(instructions[function.start + offset].opcode, target); + lengths[index] = size; +} + +__device__ void byte(uint8_t *output, U32 &cursor, uint8_t value) { + output[cursor++] = value; +} +__device__ void word(uint8_t *output, U32 &cursor, U32 value) { + for (U32 part = 0; part < 4; ++part) byte(output, cursor, uint8_t(value >> (part * 8))); +} +__device__ void x86_home(uint8_t *output, U32 &cursor, + uint8_t first, uint8_t second, uint8_t third, U32 slot) { + byte(output, cursor, first); byte(output, cursor, second); byte(output, cursor, third); + word(output, cursor, U32(-8 * (int(slot) + 1))); +} + +__device__ void emit_x86(uint8_t *output, const GPUEmitFunction &function, + const GPUEmitInstruction *instructions) { + U32 cursor = 0; + const U32 frame = (function.values * 8 + 15) & ~U32(15); + byte(output, cursor, 0x55); + byte(output, cursor, 0x48); byte(output, cursor, 0x89); byte(output, cursor, 0xe5); + byte(output, cursor, 0x48); byte(output, cursor, 0x81); byte(output, cursor, 0xec); + word(output, cursor, frame); + for (U32 offset = 0; offset < function.count; ++offset) { + const GPUEmitInstruction instruction = instructions[function.start + offset]; + switch (instruction.opcode) { + case 0: + x86_home(output, cursor, 0x48, 0x89, 0xbd, instruction.result); + break; + case 1: { + byte(output, cursor, 0x48); byte(output, cursor, 0xb8); + const uint64_t value = uint64_t(instruction.immediate); + for (U32 part = 0; part < 8; ++part) + byte(output, cursor, uint8_t(value >> (part * 8))); + x86_home(output, cursor, 0x48, 0x89, 0x85, instruction.result); + break; + } + case 2: + x86_home(output, cursor, 0x48, 0x8b, 0x85, instruction.a); + x86_home(output, cursor, 0x48, 0x89, 0x85, instruction.result); + break; + case 3: case 4: case 5: + x86_home(output, cursor, 0x48, 0x8b, 0x85, instruction.a); + x86_home(output, cursor, 0x4c, 0x8b, 0x95, instruction.b); + if (instruction.opcode == 5) { + byte(output, cursor, 0x49); byte(output, cursor, 0x0f); + byte(output, cursor, 0xaf); byte(output, cursor, 0xc2); + } else { + byte(output, cursor, 0x4c); + byte(output, cursor, instruction.opcode == 3 ? 0x01 : 0x29); + byte(output, cursor, 0xd0); + } + x86_home(output, cursor, 0x48, 0x89, 0x85, instruction.result); + break; + case 6: + x86_home(output, cursor, 0x48, 0x8b, 0x85, instruction.a); + byte(output, cursor, 0xc9); byte(output, cursor, 0xc3); + break; + } + } +} + +__device__ void emit_arm(uint8_t *output, const GPUEmitFunction &function, + const GPUEmitInstruction *instructions) { + U32 cursor = 0; + const U32 frame = (function.values * 8 + 15) & ~U32(15); + word(output, cursor, 0xa9bf7bfd); + word(output, cursor, 0x910003fd); + word(output, cursor, 0xd10003ff | (frame << 10)); + for (U32 offset = 0; offset < function.count; ++offset) { + const GPUEmitInstruction instruction = instructions[function.start + offset]; + switch (instruction.opcode) { + case 0: + word(output, cursor, 0xf90003e0 | (instruction.result << 10)); + break; + case 1: { + const uint64_t value = uint64_t(instruction.immediate); + for (U32 part = 0; part < 4; ++part) + word(output, cursor, (part == 0 ? 0xd2800009 : 0xf2800009) + | (part << 21) | (U32((value >> (part * 16)) & 0xffff) << 5)); + word(output, cursor, 0xf90003e9 | (instruction.result << 10)); + break; + } + case 2: + word(output, cursor, 0xf94003e9 | (instruction.a << 10)); + word(output, cursor, 0xf90003e9 | (instruction.result << 10)); + break; + case 3: case 4: case 5: + word(output, cursor, 0xf94003e9 | (instruction.a << 10)); + word(output, cursor, 0xf94003ea | (instruction.b << 10)); + word(output, cursor, instruction.opcode == 3 ? 0x8b0a0129 + : instruction.opcode == 4 ? 0xcb0a0129 : 0x9b0a7d29); + word(output, cursor, 0xf90003e9 | (instruction.result << 10)); + break; + case 6: + word(output, cursor, 0xf94003e0 | (instruction.a << 10)); + word(output, cursor, 0x910003bf); + word(output, cursor, 0xa8c17bfd); + word(output, cursor, 0xd65f03c0); + break; + } + } +} + +__global__ void emit_functions(const GPUEmitFunction *functions, + const GPUEmitInstruction *instructions, + const U32 *offsets, U32 function_count, + U32 target, uint8_t *output) { + const U32 index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= function_count) return; + uint8_t *destination = output + offsets[index]; + if (target == 0) emit_x86(destination, functions[index], instructions); + else emit_arm(destination, functions[index], instructions); +} + +// Every array grows only while no request is pending. Inputs are captured into +// pinned storage before submit returns, so callers can release their packets. +struct Buffer { + void *device = nullptr; + void *host = nullptr; + size_t capacity = 0; + ~Buffer() { if (device) cudaFree(device); if (host) cudaFreeHost(host); } + void abandon() { device = nullptr; host = nullptr; capacity = 0; } + void ensure(size_t bytes, GPUEmitStats &stats, bool pinned = true) { + if (bytes <= capacity) return; + if (bytes > kStorageBudget) throw std::runtime_error("buffer exceeds 512 MiB budget"); + const size_t grown = std::min(kStorageBudget, std::max(bytes, capacity * 2)); + void *new_device = nullptr, *new_host = nullptr; + check(cudaMalloc(&new_device, grown), "allocate device emission buffer"); + ++stats.buffer_allocations; + try { + if (pinned) { + check(cudaHostAlloc(&new_host, grown, cudaHostAllocDefault), "allocate pinned emission staging"); + ++stats.buffer_allocations; + } + } catch (...) { cudaFree(new_device); throw; } + void *old_device = device, *old_host = host; + device = new_device; host = new_host; capacity = grown; + if (old_device) check(cudaFree(old_device), "release prior emission buffer"); + if (old_host) check(cudaFreeHost(old_host), "release prior pinned staging"); + } + template T *gpu() { return static_cast(device); } + template T *cpu() { return static_cast(host); } + void upload(size_t bytes, cudaStream_t stream) { + check(cudaMemcpyAsync(device, host, bytes, cudaMemcpyHostToDevice, stream), "upload captured emission input"); + } + void download(size_t bytes, cudaStream_t stream) { + if (bytes) check(cudaMemcpyAsync(host, device, bytes, cudaMemcpyDeviceToHost, stream), "download initialized emission output"); + } +}; + +struct Context { + int device = 0; + cudaStream_t stream = nullptr; + cudaEvent_t begin = nullptr, counted = nullptr, scanned = nullptr; + cudaEvent_t emitted = nullptr, metadata_ready = nullptr; + Buffer functions, instructions, lengths, offsets, output, scan_storage; + std::string identity; + GPUEmitStats stats{}; + Clock::time_point start; + U32 function_count = 0; + U32 max_grid_x = 0; + size_t output_capacity = 0; + uint64_t submissions = 0; + bool pending = false, poisoned = false; + + // Failed synchronization is not used as proof that host staging is safe to + // free. An unresolved failed context leaks its owned allocations rather + // than freeing storage still potentially used by a kernel or DMA operation. + bool quiesce() noexcept { + if (cudaSetDevice(device) != cudaSuccess) { poisoned = true; return false; } + if (stream && cudaStreamSynchronize(stream) != cudaSuccess) { poisoned = true; return false; } + pending = false; + return true; + } + ~Context() { + if (!quiesce()) { + functions.abandon(); instructions.abandon(); lengths.abandon(); + offsets.abandon(); output.abandon(); scan_storage.abandon(); + return; + } + for (cudaEvent_t event : {begin, counted, scanned, emitted, metadata_ready}) + if (event) cudaEventDestroy(event); + if (stream) cudaStreamDestroy(stream); + } + void select() { + check(cudaSetDevice(device), "select owning CUDA emission device"); + if (poisoned) throw std::runtime_error("CUDA emission context is poisoned after an unresolved device failure"); + } +}; + +void validate_dimensions(const GPUEmitInput &input) { + if (input.reserved != 0 || input.target > 1 || input.function_count == 0 + || input.instruction_count == 0 || input.function_count > U32(INT_MAX) + || !input.functions || !input.instructions) + throw std::runtime_error("invalid scalar emission dimensions, target, reserved field, or input pointer"); + array_bytes(input.function_count, sizeof(GPUEmitFunction)); + array_bytes(input.instruction_count, sizeof(GPUEmitInstruction)); + const uint64_t bound = 12ULL * input.function_count + 25ULL * input.instruction_count; + if (input.output_capacity < bound || input.output_capacity > kStorageBudget) + throw std::runtime_error("output capacity must cover the bounded scalar emission plan and fit 512 MiB"); +} + +void validate(const GPUEmitInput &input) { + std::vector symbols(input.function_count, 0); + uint64_t next = 0; + for (U32 index = 0; index < input.function_count; ++index) { + const GPUEmitFunction &function = input.functions[index]; + const uint64_t end = uint64_t(function.start) + function.count; + if (function.start != next || function.count == 0 || end > input.instruction_count + || function.values == 0 || function.values > 256 || function.values > function.count + || function.symbol >= input.function_count || symbols[function.symbol]++) + throw std::runtime_error("function ranges, symbols, or scalar frame are invalid"); + bool defined[256] = {}; + U32 definitions = 0; + for (U32 offset = 0; offset < function.count; ++offset) { + const GPUEmitInstruction &instruction = input.instructions[function.start + offset]; + if (instruction.opcode > 6 || ((instruction.opcode == 6) != (offset + 1 == function.count))) + throw std::runtime_error("function must end with its only Return and use supported opcodes"); + const U32 operands = instruction.opcode <= 1 ? 0 : instruction.opcode == 2 || instruction.opcode == 6 ? 1 : 2; + if ((operands >= 1 && (instruction.a >= function.values || !defined[instruction.a])) + || (operands == 2 && (instruction.b >= function.values || !defined[instruction.b]))) + throw std::runtime_error("instruction reads an undefined scalar value"); + if ((operands == 0 && instruction.a != 0) || (operands < 2 && instruction.b != 0) + || (instruction.opcode != 1 && instruction.immediate != 0)) + throw std::runtime_error("unused instruction operands must be zero; only argument zero is supported"); + if (instruction.opcode == 6) { + if (instruction.result != UINT32_MAX) + throw std::runtime_error("Return cannot define a scalar value"); + } else { + if (instruction.result >= function.values || defined[instruction.result]) + throw std::runtime_error("instruction result is invalid or already defined"); + defined[instruction.result] = true; + ++definitions; + } + } + if (definitions != function.values) + throw std::runtime_error("scalar frame contains undefined value slots"); + next = end; + } + if (next != input.instruction_count) + throw std::runtime_error("instruction array has records outside function ranges"); +} +float event_ms(cudaEvent_t begin, cudaEvent_t end) { + float milliseconds = 0; + check(cudaEventElapsedTime(&milliseconds, begin, end), "measure CUDA emission event interval"); + return milliseconds; +} +} // namespace + +extern "C" void *gpuemit_create(const char *, U32 device_index, char *error, size_t capacity) { + try { + if (device_index > U32(INT_MAX)) throw std::runtime_error("CUDA device index is out of range"); + int count = 0; + check(cudaGetDeviceCount(&count), "enumerate physical CUDA devices"); + if (device_index >= U32(count)) throw std::runtime_error("requested CUDA emission device is unavailable"); + std::unique_ptr context(new Context); + context->device = int(device_index); + context->select(); + cudaDeviceProp properties{}; + check(cudaGetDeviceProperties(&properties, context->device), "read physical CUDA device properties and UUID"); + context->max_grid_x = U32(properties.maxGridSize[0]); + char identity[64] = {}; + const auto *u = reinterpret_cast(properties.uuid.bytes); + std::snprintf(identity, sizeof(identity), + "CUDA:%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + u[0],u[1],u[2],u[3],u[4],u[5],u[6],u[7],u[8],u[9],u[10],u[11],u[12],u[13],u[14],u[15]); + context->identity = identity; + check(cudaStreamCreateWithFlags(&context->stream, cudaStreamNonBlocking), "create persistent emission stream"); + for (cudaEvent_t *event : {&context->begin, &context->counted, &context->scanned, + &context->emitted, &context->metadata_ready}) + check(cudaEventCreate(event), "create persistent emission timing event"); + diagnostic(error, capacity, ""); + return context.release(); + } catch (const std::exception &e) { diagnostic(error, capacity, e.what()); return nullptr; } +} + +extern "C" int gpuemit_device_id(void *pointer, char *output, size_t capacity) { + if (!pointer || !output || capacity == 0) return -1; + auto &context = *static_cast(pointer); + try { + context.select(); + if (capacity <= context.identity.size()) { output[0] = '\0'; return -1; } + std::memcpy(output, context.identity.c_str(), context.identity.size() + 1); + return 0; + } catch (const std::exception &e) { diagnostic(output, capacity, e.what()); return -1; } +} + +extern "C" int gpuemit_submit(void *pointer, const GPUEmitInput *input, char *error, size_t capacity) { + if (!pointer || !input) { diagnostic(error, capacity, "null CUDA emission context or input"); return -1; } + auto &context = *static_cast(pointer); + try { + const auto entry_time = Clock::now(); + context.select(); + // This error must not cancel the already submitted request. + if (context.pending) { diagnostic(error, capacity, "a CUDA emission request is already pending"); return -1; } + context.start = entry_time; + validate_dimensions(*input); + const U32 blocks = (input->function_count + kThreads - 1) / kThreads; + if (blocks > context.max_grid_x) throw std::runtime_error("emission batch exceeds this device's kernel grid capacity"); + context.stats = {}; + context.stats.reused_pipeline = context.submissions != 0; + const size_t function_bytes = array_bytes(input->function_count, sizeof(GPUEmitFunction)); + const size_t instruction_bytes = array_bytes(input->instruction_count, sizeof(GPUEmitInstruction)); + const size_t index_bytes = array_bytes(input->function_count, sizeof(U32)); + context.functions.ensure(function_bytes, context.stats); + context.instructions.ensure(instruction_bytes, context.stats); + context.lengths.ensure(index_bytes, context.stats); + context.offsets.ensure(index_bytes, context.stats); + context.output.ensure(size_t(input->output_capacity), context.stats); + size_t scan_bytes = 0; + check(cub::DeviceScan::ExclusiveSum(nullptr, scan_bytes, + context.lengths.gpu(), context.offsets.gpu(), int(input->function_count), context.stream), "size CUB device prefix scan"); + // CUB's null pointer denotes the size-query mode even if a future scan + // implementation needs no scratch bytes. Execute with a nonnull pointer. + context.scan_storage.ensure(std::max(size_t(1), scan_bytes), context.stats, false); + std::memcpy(context.functions.host, input->functions, function_bytes); + std::memcpy(context.instructions.host, input->instructions, instruction_bytes); + GPUEmitInput snapshot = *input; + snapshot.functions = context.functions.cpu(); + snapshot.instructions = context.instructions.cpu(); + // Validate the captured records that the GPU will actually consume, + // rather than rereading caller storage between validation and upload. + validate(snapshot); + context.function_count = input->function_count; + context.output_capacity = size_t(input->output_capacity); + context.pending = true; + context.functions.upload(function_bytes, context.stream); + context.instructions.upload(instruction_bytes, context.stream); + check(cudaEventRecord(context.begin, context.stream), "record emission kernel start"); + count_functions<<>>( + context.functions.gpu(), context.instructions.gpu(), + input->function_count, input->target, context.lengths.gpu()); + check(cudaGetLastError(), "launch scalar instruction-count kernel"); + check(cudaEventRecord(context.counted, context.stream), "record instruction-count completion"); + check(cub::DeviceScan::ExclusiveSum(context.scan_storage.device, scan_bytes, + context.lengths.gpu(), context.offsets.gpu(), int(input->function_count), context.stream), "scan function emission offsets on device"); + check(cudaEventRecord(context.scanned, context.stream), "record prefix-scan completion"); + emit_functions<<>>( + context.functions.gpu(), context.instructions.gpu(), + context.offsets.gpu(), input->function_count, input->target, context.output.gpu()); + check(cudaGetLastError(), "launch scalar machine-code emission kernel"); + check(cudaEventRecord(context.emitted, context.stream), "record machine-code emission completion"); + context.lengths.download(index_bytes, context.stream); + context.offsets.download(index_bytes, context.stream); + check(cudaEventRecord(context.metadata_ready, context.stream), "record emission-length download completion"); + // Counts only the two explicit application kernels. CUB may internally + // launch additional kernels, all included in gpu_ms, not in this count. + context.stats.kernel_dispatches = 2; + ++context.submissions; + diagnostic(error, capacity, ""); + return 0; + } catch (const std::exception &e) { + if (context.pending) context.quiesce(); + diagnostic(error, capacity, e.what()); + return -1; + } +} + +extern "C" int gpuemit_finish(void *pointer, uint8_t *output, size_t output_capacity, + U32 *offsets, U32 *lengths, size_t function_count, + GPUEmitStats *stats, char *error, size_t capacity) { + if (!pointer) { diagnostic(error, capacity, "null CUDA emission context"); return -1; } + auto &context = *static_cast(pointer); + try { + context.select(); + if (!context.pending) { diagnostic(error, capacity, "no CUDA emission request is pending"); return -1; } + // A malformed result destination leaves the request intact for retry. + if (!output || !offsets || !lengths || !stats || output_capacity < context.output_capacity + || function_count < context.function_count) { + diagnostic(error, capacity, "CUDA emission result destinations are null or too short"); return -1; + } + check(cudaEventSynchronize(context.metadata_ready), "wait for emitted code and output lengths"); + const U32 *device_offsets = context.offsets.cpu(); + const U32 *device_lengths = context.lengths.cpu(); + uint64_t written = 0; + for (U32 index = 0; index < context.function_count; ++index) { + const auto function = context.functions.cpu()[index]; + if (device_offsets[index] != written || device_lengths[index] == 0 + || device_lengths[index] > 12ULL + 25ULL * function.count) + throw std::runtime_error("GPU instruction counts or prefix offsets violate the output plan"); + written += device_lengths[index]; + if (written > context.output_capacity) + throw std::runtime_error("GPU-written code exceeds the retained output capacity"); + } + // Never download the unused capacity: every byte in [0, written) was + // initialized by its function's emitter after the device prefix scan. + context.output.download(size_t(written), context.stream); + check(cudaStreamSynchronize(context.stream), "wait for initialized machine-code download"); + context.stats.count_gpu_ms = event_ms(context.begin, context.counted); + context.stats.emit_gpu_ms = event_ms(context.scanned, context.emitted); + context.stats.gpu_ms = event_ms(context.begin, context.emitted); + context.stats.host_prefix_ms = 0; + context.stats.output_bytes = written; + std::memcpy(output, context.output.host, size_t(written)); + const size_t index_bytes = array_bytes(context.function_count, sizeof(U32)); + std::memcpy(offsets, context.offsets.host, index_bytes); + std::memcpy(lengths, context.lengths.host, index_bytes); + context.stats.total_ms = elapsed(context.start); + *stats = context.stats; + context.pending = false; + diagnostic(error, capacity, ""); + return 0; + } catch (const std::exception &e) { + context.quiesce(); + diagnostic(error, capacity, e.what()); + return -1; + } +} + +extern "C" void gpuemit_destroy(void *pointer) { + delete static_cast(pointer); +} diff --git a/experiments/gpu-rust-compiler/native/codegen_metal.mm b/experiments/gpu-rust-compiler/native/codegen_metal.mm new file mode 100644 index 00000000000..5aa59c78ef9 --- /dev/null +++ b/experiments/gpu-rust-compiler/native/codegen_metal.mm @@ -0,0 +1,390 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#import +#import + +#include "codegen_bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static_assert(sizeof(GPUEmitInstruction) == 24 && offsetof(GPUEmitInstruction, immediate) == 16, + "Instruction shader ABI changed"); +static_assert(sizeof(GPUEmitFunction) == 16 && sizeof(GPUEmitStats) == 64, + "Function or statistics ABI changed"); + +namespace { +using Clock = std::chrono::steady_clock; +constexpr uint64_t Budget = 512ULL * 1024 * 1024; +constexpr uint32_t MaxFunctions = 1024 * 1024; +constexpr uint32_t MaxInstructions = 16 * 1024 * 1024; +enum Buffer { Functions, Instructions, Lengths, Offsets, Output, Statuses, BufferCount }; +struct Parameters { uint32_t functions, target, output_capacity, reserved; }; + +struct Context { + id device; + id queue; + id library; + id count_pipeline, emit_pipeline; + id buffers[BufferCount] = {}; + size_t capacities[BufferCount] = {}; + id pending; + std::mutex mutex; + Clock::time_point started; + uint32_t functions = 0, target = 0, allocations = 0, reused = 0; + size_t output_capacity = 0; + uint64_t submitted = 0; +}; + +void diagnostic(char *error, size_t capacity, const std::string &message) { + if (error && capacity) std::snprintf(error, capacity, "%s", message.c_str()); +} + +std::string metal_error(const char *operation, NSError *error) { + return std::string(operation) + ": " + + (error.localizedDescription.UTF8String ?: "Metal supplied no diagnostic"); +} + +bool validate(const GPUEmitInput *input, const std::vector &functions, + const std::vector &instructions, std::string &error) { + if (input->target > 1 || input->reserved != 0) { + error = "Target must be x86-64 SysV (0) or AArch64 (1), with reserved zero"; + return false; + } + uint64_t next = 0, maximum_output = 0; + for (const auto &function : functions) { + if (function.start != next || !function.count || function.count > 257 || + !function.values || function.values > 256 || + function.count > instructions.size() - next) { + error = "Functions must densely cover instructions with 1..256 values and 1..257 operations"; + return false; + } + bool defined[256] = {}; + for (uint32_t position = 0; position < function.count; ++position) { + const auto &instruction = instructions[size_t(next) + position]; + if (instruction.opcode > 6 || + ((instruction.opcode == 6) != (position + 1 == function.count))) { + error = "Each function must end in its only return and use supported opcodes"; + return false; + } + const uint32_t count = instruction.opcode <= 1 ? 0 : + (instruction.opcode == 2 || instruction.opcode == 6 ? 1 : 2); + const uint32_t operands[2] = {instruction.a, instruction.b}; + for (uint32_t operand = 0; operand < count; ++operand) { + if (operands[operand] >= function.values || !defined[operands[operand]]) { + error = "An instruction reads an undefined or out-of-range value"; + return false; + } + } + if (instruction.opcode != 6) { + if (instruction.result >= function.values || defined[instruction.result]) { + error = "Instruction results must define distinct in-range values"; + return false; + } + defined[instruction.result] = true; + } + if (instruction.opcode == 0 && instruction.immediate != 0) { + error = "Only argument index zero is supported"; + return false; + } + } + next += function.count; + maximum_output += 12 + 25ULL * function.count; + } + if (next != instructions.size()) { + error = "Function coverage does not match instruction_count"; + return false; + } + if (maximum_output > Budget || !input->output_capacity || input->output_capacity > Budget) { + error = "Output capacity and worst-case output must fit the 512 MiB budget"; + return false; + } + return true; +} + +bool ensure_buffer(Context &context, Buffer index, size_t bytes, + const char *label, std::string &error) { + bytes = std::max(bytes, size_t(4)); + if (bytes > context.device.maxBufferLength || bytes > Budget) { + error = std::string(label) + " exceeds the Metal buffer or experiment budget"; + return false; + } + if (context.capacities[index] >= bytes) return true; + size_t capacity = std::max(context.capacities[index], size_t(4)); + const size_t maximum = std::min(size_t(context.device.maxBufferLength), size_t(Budget)); + while (capacity < bytes && capacity <= maximum / 2) capacity *= 2; + capacity = std::max(capacity, bytes); + id buffer = [context.device newBufferWithLength:capacity + options:MTLResourceStorageModeShared]; + if (!buffer || !buffer.contents) { + error = std::string("Could not allocate Metal buffer for ") + label; + return false; + } + buffer.label = [NSString stringWithUTF8String:label]; + context.buffers[index] = buffer; + context.capacities[index] = capacity; + ++context.allocations; + return true; +} + +void dispatch(id encoder, id pipeline, + uint32_t functions) { + const NSUInteger threads = std::max(NSUInteger(1), + std::min(pipeline.threadExecutionWidth, pipeline.maxTotalThreadsPerThreadgroup)); + [encoder dispatchThreads:MTLSizeMake(functions, 1, 1) + threadsPerThreadgroup:MTLSizeMake(threads, 1, 1)]; +} + +bool completed(id command, double &gpu_ms, std::string &error) { + [command waitUntilCompleted]; + if (command.status != MTLCommandBufferStatusCompleted) { + error = metal_error("Metal code generation command failed", command.error); + return false; + } + gpu_ms = (command.GPUEndTime - command.GPUStartTime) * 1000.0; + if (!std::isfinite(gpu_ms) || gpu_ms <= 0) { + error = "Metal GPU timestamps are unavailable; physical execution time is unverified"; + return false; + } + return true; +} + +int submit(Context &context, const GPUEmitInput *input, char *error, size_t capacity) { + std::lock_guard lock(context.mutex); + if (context.pending) { + diagnostic(error, capacity, "One request is already pending; finish it before submitting another"); + return -1; + } + const auto started = Clock::now(); + if (!input || !input->functions || !input->instructions || !input->function_count || + !input->instruction_count || input->function_count > MaxFunctions || + input->instruction_count > MaxInstructions) { + diagnostic(error, capacity, "Input arrays/counts must be nonempty and within the declared budgets"); + return -1; + } + // Validate an owned snapshot and then copy that same snapshot into device- + // owned shared buffers. Caller mutation after submit cannot change work. + const std::vector functions(input->functions, + input->functions + input->function_count); + const std::vector instructions(input->instructions, + input->instructions + input->instruction_count); + std::string detail; + if (!validate(input, functions, instructions, detail)) { + diagnostic(error, capacity, detail); return -1; + } + const size_t bytes[BufferCount] = { + functions.size() * sizeof(GPUEmitFunction), instructions.size() * sizeof(GPUEmitInstruction), + functions.size() * sizeof(uint32_t), functions.size() * sizeof(uint32_t), + size_t(input->output_capacity), functions.size() * sizeof(uint32_t) + }; + const char *labels[BufferCount] = {"codegen functions", "codegen instructions", "GPU code lengths", + "CPU exclusive offsets", "GPU machine code", "GPU emit status"}; + context.allocations = 0; + for (uint32_t index = 0; index < BufferCount; ++index) { + if (!ensure_buffer(context, Buffer(index), bytes[index], labels[index], detail)) { + diagnostic(error, capacity, detail); return -1; + } + } + std::memcpy(context.buffers[Functions].contents, functions.data(), bytes[Functions]); + std::memcpy(context.buffers[Instructions].contents, instructions.data(), bytes[Instructions]); + std::memset(context.buffers[Lengths].contents, 0, bytes[Lengths]); + std::memset(context.buffers[Statuses].contents, 0, bytes[Statuses]); + id command = [context.queue commandBuffer]; + if (!command) { diagnostic(error, capacity, "Could not create GPU count command"); return -1; } + id encoder = [command computeCommandEncoder]; + if (!encoder) { diagnostic(error, capacity, "Could not create GPU count encoder"); return -1; } + command.label = @"GPU baseline machine-code size counting"; + [encoder setComputePipelineState:context.count_pipeline]; + [encoder setBuffer:context.buffers[Functions] offset:0 atIndex:0]; + [encoder setBuffer:context.buffers[Instructions] offset:0 atIndex:1]; + [encoder setBuffer:context.buffers[Lengths] offset:0 atIndex:2]; + const Parameters parameters = {input->function_count, input->target, uint32_t(input->output_capacity), 0}; + [encoder setBytes:¶meters length:sizeof(parameters) atIndex:3]; + dispatch(encoder, context.count_pipeline, input->function_count); + [encoder endEncoding]; + context.functions = input->function_count; + context.target = input->target; + context.output_capacity = size_t(input->output_capacity); + context.started = started; + context.reused = context.submitted ? 1 : 0; + context.pending = command; + ++context.submitted; + [command commit]; + return 0; +} + +int finish(Context &context, uint8_t *output, size_t output_capacity, + uint32_t *offsets, uint32_t *lengths, size_t function_count, + GPUEmitStats *stats, char *error, size_t capacity) { + std::lock_guard lock(context.mutex); + if (!context.pending) { diagnostic(error, capacity, "No pending code generation request"); return -1; } + if (!output || output_capacity < context.output_capacity || !offsets || !lengths || + function_count < context.functions) { + diagnostic(error, capacity, "Output/offset/length buffers are short or NULL; pending request retained"); + return -1; + } + std::string detail; + double count_gpu_ms = 0, emit_gpu_ms = 0; + if (!completed(context.pending, count_gpu_ms, detail)) { + context.pending = nil; diagnostic(error, capacity, detail); return -1; + } + const auto prefix_started = Clock::now(); + const uint32_t *gpu_lengths = static_cast(context.buffers[Lengths].contents); + uint32_t *host_offsets = static_cast(context.buffers[Offsets].contents); + const auto *functions = static_cast(context.buffers[Functions].contents); + uint64_t total = 0; + for (uint32_t index = 0; index < context.functions; ++index) { + if (!gpu_lengths[index] || gpu_lengths[index] > 12 + 25ULL * functions[index].count || + gpu_lengths[index] > context.output_capacity - total) { + context.pending = nil; + diagnostic(error, capacity, "GPU counts exceed the declared output capacity or function budget"); + return -1; + } + host_offsets[index] = uint32_t(total); + total += gpu_lengths[index]; + } + const double prefix_ms = std::chrono::duration(Clock::now() - prefix_started).count(); + // Prefix offsets are a CPU stage in this implementation. Actual lengths + // originate in the count kernel; the emit kernel produces every code byte. + id command = [context.queue commandBuffer]; + if (!command) { context.pending = nil; diagnostic(error, capacity, "Could not create GPU emit command"); return -1; } + id encoder = [command computeCommandEncoder]; + if (!encoder) { context.pending = nil; diagnostic(error, capacity, "Could not create GPU emit encoder"); return -1; } + command.label = @"GPU baseline native machine-code emission"; + [encoder setComputePipelineState:context.emit_pipeline]; + [encoder setBuffer:context.buffers[Functions] offset:0 atIndex:0]; + [encoder setBuffer:context.buffers[Instructions] offset:0 atIndex:1]; + [encoder setBuffer:context.buffers[Offsets] offset:0 atIndex:2]; + [encoder setBuffer:context.buffers[Lengths] offset:0 atIndex:3]; + [encoder setBuffer:context.buffers[Output] offset:0 atIndex:4]; + [encoder setBuffer:context.buffers[Statuses] offset:0 atIndex:5]; + const Parameters parameters = {context.functions, context.target, uint32_t(context.output_capacity), 0}; + [encoder setBytes:¶meters length:sizeof(parameters) atIndex:6]; + dispatch(encoder, context.emit_pipeline, context.functions); + [encoder endEncoding]; + context.pending = command; + [command commit]; + const bool success = completed(command, emit_gpu_ms, detail); + context.pending = nil; + if (!success) { diagnostic(error, capacity, detail); return -1; } + const uint32_t *statuses = static_cast(context.buffers[Statuses].contents); + for (uint32_t index = 0; index < context.functions; ++index) { + if (statuses[index] != 1) { + diagnostic(error, capacity, "GPU emission did not match its counted function length"); + return -1; + } + } + std::memcpy(output, context.buffers[Output].contents, size_t(total)); + std::memcpy(offsets, host_offsets, size_t(context.functions) * sizeof(uint32_t)); + std::memcpy(lengths, gpu_lengths, size_t(context.functions) * sizeof(uint32_t)); + if (stats) { + stats->gpu_ms = count_gpu_ms + emit_gpu_ms; + stats->count_gpu_ms = count_gpu_ms; + stats->emit_gpu_ms = emit_gpu_ms; + stats->host_prefix_ms = prefix_ms; + stats->total_ms = std::chrono::duration(Clock::now() - context.started).count(); + stats->kernel_dispatches = 2; + stats->output_bytes = total; + stats->buffer_allocations = context.allocations; + stats->reused_pipeline = context.reused; + } + return 0; +} +} // namespace + +extern "C" void *gpuemit_create(const char *shader_path, uint32_t device_index, + char *error, size_t capacity) { + @autoreleasepool { + diagnostic(error, capacity, ""); + if (!shader_path || !*shader_path) { + diagnostic(error, capacity, "Metal shader path is required"); return nullptr; + } + try { + NSArray> *devices = MTLCopyAllDevices(); + if (device_index >= devices.count) { + diagnostic(error, capacity, "Metal device index is unavailable; physical device count is " + std::to_string(devices.count)); + return nullptr; + } + id device = devices[device_index]; + NSError *detail = nil; + NSString *path = [NSString stringWithUTF8String:shader_path]; + if (!path) { diagnostic(error, capacity, "Shader path is not UTF-8"); return nullptr; } + NSString *source = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:&detail]; + if (!source) { diagnostic(error, capacity, metal_error("Could not read codegen shader", detail)); return nullptr; } + id library = [device newLibraryWithSource:source options:nil error:&detail]; + if (!library) { diagnostic(error, capacity, metal_error("Could not compile codegen shader", detail)); return nullptr; } + id count = [library newFunctionWithName:@"gpuemit_count"]; + id emit = [library newFunctionWithName:@"gpuemit_emit"]; + if (!count || !emit) { diagnostic(error, capacity, "Shader must expose gpuemit_count and gpuemit_emit"); return nullptr; } + id count_pipeline = [device newComputePipelineStateWithFunction:count error:&detail]; + if (!count_pipeline) { diagnostic(error, capacity, metal_error("Could not create count pipeline", detail)); return nullptr; } + id emit_pipeline = [device newComputePipelineStateWithFunction:emit error:&detail]; + if (!emit_pipeline) { diagnostic(error, capacity, metal_error("Could not create emit pipeline", detail)); return nullptr; } + id queue = [device newCommandQueue]; + if (!queue) { diagnostic(error, capacity, "Could not create Metal codegen queue"); return nullptr; } + Context *context = new Context; + context->device = device; context->queue = queue; context->library = library; + context->count_pipeline = count_pipeline; context->emit_pipeline = emit_pipeline; + return context; + } catch (const std::exception &exception) { + diagnostic(error, capacity, std::string("Metal creation failed: ") + exception.what()); + return nullptr; + } + } +} + +extern "C" int gpuemit_device_id(void *context, char *output, size_t capacity) { + @autoreleasepool { + if (!context || !output || !capacity) return -1; + const auto *owned = static_cast(context); + const int bytes = std::snprintf(output, capacity, "metal:%016llx:%s", + static_cast(owned->device.registryID), owned->device.name.UTF8String ?: "unknown"); + return bytes >= 0 && size_t(bytes) < capacity ? 0 : -1; + } +} + +extern "C" int gpuemit_submit(void *context, const GPUEmitInput *input, char *error, size_t capacity) { + @autoreleasepool { + diagnostic(error, capacity, ""); + if (!context) { diagnostic(error, capacity, "Metal codegen context is NULL"); return -1; } + try { return submit(*static_cast(context), input, error, capacity); } + catch (const std::exception &exception) { + diagnostic(error, capacity, std::string("Metal submission failed: ") + exception.what()); return -1; + } + } +} + +extern "C" int gpuemit_finish(void *context, uint8_t *output, size_t output_capacity, + uint32_t *offsets, uint32_t *lengths, size_t function_count, + GPUEmitStats *stats, char *error, size_t capacity) { + @autoreleasepool { + diagnostic(error, capacity, ""); + if (stats) std::memset(stats, 0, sizeof(*stats)); + if (!context) { diagnostic(error, capacity, "Metal codegen context is NULL"); return -1; } + try { return finish(*static_cast(context), output, output_capacity, + offsets, lengths, function_count, stats, error, capacity); } + catch (const std::exception &exception) { + diagnostic(error, capacity, std::string("Metal finish failed: ") + exception.what()); return -1; + } + } +} + +extern "C" void gpuemit_destroy(void *context) { + @autoreleasepool { + if (!context) return; + auto *owned = static_cast(context); + { + std::lock_guard lock(owned->mutex); + if (owned->pending) [owned->pending waitUntilCompleted]; + owned->pending = nil; + } + delete owned; + } +} diff --git a/experiments/gpu-rust-compiler/native/cuda_bridge.cu b/experiments/gpu-rust-compiler/native/cuda_bridge.cu new file mode 100644 index 00000000000..f93d9b43caa --- /dev/null +++ b/experiments/gpu-rust-compiler/native/cuda_bridge.cu @@ -0,0 +1,421 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "cuda_bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using Clock = std::chrono::steady_clock; +using U32 = uint32_t; +using U64 = unsigned long long; +static_assert(sizeof(GPULabCudaGroup) == 20, "CUDA group ABI changed"); +struct Progress { U64 visits, edges, bits; U32 sweeps, converged; }; + +double elapsed(Clock::time_point start) { + return std::chrono::duration(Clock::now() - start).count(); +} +void check(cudaError_t status, const char *operation) { + if (status != cudaSuccess) throw std::runtime_error(std::string(operation) + ": " + cudaGetErrorString(status)); +} +void error(char *out, size_t count, const char *message) { + if (out && count) std::snprintf(out, count, "%s", message); +} + +/* Each CUDA block owns exactly one function-word pair. Dense Jacobi uses a + * pair of immutable/current snapshots. Every thread reaches each barrier and + * the shared termination flag is read only after the writers synchronize. + */ +__global__ void dense(const GPULabCudaGroup *groups, const U32 *offsets, + const U32 *successors, const U32 *uses, const U32 *defs, + const U32 *phi, U32 *result, Progress *progress) { + const auto group = groups[blockIdx.x]; + const U32 n = group.block_count; + extern __shared__ U32 storage[]; + U32 *current = storage, *next = current + n; + __shared__ U32 changed; + for (U32 row = threadIdx.x; row < n; row += blockDim.x) current[row] = 0; + __syncthreads(); + U32 sweeps = 0; + bool converged = false; + for (U32 round = 0; round <= n; ++round) { + if (threadIdx.x == 0) changed = 0; + __syncthreads(); + for (U32 row = threadIdx.x; row < n; row += blockDim.x) { + const U32 block = group.block_base + row; + const U32 cell = group.row_base + row * group.words + group.word; + U32 out = phi[cell]; + for (U32 e = offsets[block]; e < offsets[block + 1]; ++e) + out |= current[successors[e] - group.block_base]; + const U32 value = uses[cell] | (out & ~defs[cell]); + next[row] = value; + if (value != current[row]) atomicOr(&changed, 1U); + } + __syncthreads(); + ++sweeps; + U32 *swap = current; current = next; next = swap; + if (!changed) { converged = true; break; } + } + U64 local_bits = 0; + for (U32 row = threadIdx.x; row < n; row += blockDim.x) { + result[group.row_base + row * group.words + group.word] = current[row]; + local_bits += __popc(current[row]); + } + __shared__ U64 bits; + if (threadIdx.x == 0) bits = 0; + __syncthreads(); + atomicAdd(&bits, local_bits); + __syncthreads(); + if (threadIdx.x == 0) progress[blockIdx.x] = { + U64(n) * sweeps, + U64(offsets[group.block_base + n] - offsets[group.block_base]) * sweeps, + bits, sweeps, converged ? 1U : 0U}; +} + +/* Delta frontier proof: seed GEN = USE | (PHIOUT & ~DEF). A source frontier + * contains only bits that first appeared in that row in the previous round. + * atomicOr(values[p], delta & ~DEF[p]) assigns each new bit to exactly one + * predecessor update; those new bits are atomicOr'ed into the next frontier. + * Concurrent parents cannot lose facts. Frontier clearing precedes propagation + * and a uniform barrier precedes reading it. No queue capacity or producer / + * consumer termination race exists. Each bit advances along a shortest path + * of at most n-1 edges, so n+1 rounds include a final empty frontier check. + */ +__global__ void sparse(const GPULabCudaGroup *groups, const U32 *offsets, + const U32 *predecessors, const U32 *uses, const U32 *defs, + const U32 *phi, U32 *result, Progress *progress) { + const auto group = groups[blockIdx.x]; + const U32 n = group.block_count; + extern __shared__ U32 storage[]; + U32 *values = storage, *frontier = values + n, *next = frontier + n; + __shared__ U32 changed; + __shared__ U64 visits, edges; + U64 local_visits = 0, local_edges = 0; + if (threadIdx.x == 0) { visits = 0; edges = 0; } + for (U32 row = threadIdx.x; row < n; row += blockDim.x) { + const U32 cell = group.row_base + row * group.words + group.word; + values[row] = frontier[row] = uses[cell] | (phi[cell] & ~defs[cell]); + } + __syncthreads(); + U32 sweeps = 0; + bool converged = false; + for (U32 round = 0; round <= n; ++round) { + if (threadIdx.x == 0) changed = 0; + for (U32 row = threadIdx.x; row < n; row += blockDim.x) next[row] = 0; + __syncthreads(); + for (U32 row = threadIdx.x; row < n; row += blockDim.x) { + const U32 delta = frontier[row]; + if (!delta) continue; + ++local_visits; + const U32 block = group.block_base + row; + for (U32 e = offsets[block]; e < offsets[block + 1]; ++e) { + const U32 pred = predecessors[e] - group.block_base; + const U32 cell = group.row_base + pred * group.words + group.word; + const U32 candidate = delta & ~defs[cell]; + if (!candidate) continue; + const U32 old = atomicOr(&values[pred], candidate); + const U32 added = candidate & ~old; + if (added) { atomicOr(&next[pred], added); atomicOr(&changed, 1U); } + } + local_edges += offsets[block + 1] - offsets[block]; + } + __syncthreads(); + ++sweeps; + U32 *swap = frontier; frontier = next; next = swap; + if (!changed) { converged = true; break; } + } + U64 local_bits = 0; + for (U32 row = threadIdx.x; row < n; row += blockDim.x) { + result[group.row_base + row * group.words + group.word] = values[row]; + local_bits += __popc(values[row]); + } + __shared__ U64 bits; + if (threadIdx.x == 0) bits = 0; + __syncthreads(); + atomicAdd(&bits, local_bits); + atomicAdd(&visits, local_visits); + atomicAdd(&edges, local_edges); + __syncthreads(); + if (threadIdx.x == 0) progress[blockIdx.x] = {visits, edges, bits, sweeps, converged ? 1U : 0U}; +} + +struct Buffer { + void *device = nullptr, *host = nullptr; + size_t capacity = 0; + ~Buffer() { if (device) cudaFree(device); if (host) cudaFreeHost(host); } + void ensure(size_t bytes, GPULabCudaStats &stats) { + if (bytes <= capacity) return; + const size_t grown = capacity > std::numeric_limits::max() / 2 ? bytes : std::max(bytes, capacity * 2); + void *new_device = nullptr, *new_host = nullptr; + check(cudaMalloc(&new_device, grown), "allocate CUDA device buffer"); + try { check(cudaHostAlloc(&new_host, grown, cudaHostAllocDefault), "allocate CUDA pinned staging"); } + catch (...) { cudaFree(new_device); throw; } + if (device) cudaFree(device); + if (host) cudaFreeHost(host); + device = new_device; host = new_host; capacity = grown; + ++stats.device_allocations; ++stats.pinned_allocations; + } + template T *gpu() { return static_cast(device); } + template T *cpu() { return static_cast(host); } + void capture(const void *source, size_t bytes) { if (bytes) std::memcpy(host, source, bytes); } + void upload(size_t bytes, cudaStream_t stream) { if (bytes) check(cudaMemcpyAsync(device, host, bytes, cudaMemcpyHostToDevice, stream), "upload CUDA input"); } + void download(size_t bytes, cudaStream_t stream) { if (bytes) check(cudaMemcpyAsync(host, device, bytes, cudaMemcpyDeviceToHost, stream), "download CUDA output"); } +}; + +struct Bucket { U32 offset, count, max_blocks; }; +struct Context { + cudaStream_t stream = nullptr; + cudaEvent_t begin = nullptr, uploaded = nullptr, solved = nullptr, done = nullptr; + GPULabCudaCapabilities capabilities{}; + cudaDeviceProp properties{}; + Buffer groups, successors_offsets, successors, predecessors_offsets, predecessors, uses, defs, phi, output, progress; + GPULabCudaInput dimensions{}; + GPULabCudaStats stats{}; + std::vector buckets; + Clock::time_point start; + double initialization_ms = 0; + size_t submissions = 0; + bool in_flight = false, resident = false; + ~Context() { + if (stream) cudaStreamSynchronize(stream); + if (begin) cudaEventDestroy(begin); + if (uploaded) cudaEventDestroy(uploaded); + if (solved) cudaEventDestroy(solved); + if (done) cudaEventDestroy(done); + if (stream) cudaStreamDestroy(stream); + } + void initialize() { + const auto started = Clock::now(); + check(cudaFree(nullptr), "initialize CUDA runtime"); + int device = 0; check(cudaGetDevice(&device), "query CUDA device"); + check(cudaGetDeviceProperties(&properties, device), "query CUDA properties"); + std::snprintf(capabilities.device_name, sizeof(capabilities.device_name), "%s", properties.name); + char *uuid = capabilities.device_uuid; + for (size_t i = 0; i < sizeof(properties.uuid.bytes); ++i) { + std::snprintf(uuid + i * 2, sizeof(capabilities.device_uuid) - i * 2, + "%02x", static_cast(properties.uuid.bytes[i])); + } + check(cudaDriverGetVersion(&capabilities.driver_version), "query CUDA driver version"); + check(cudaRuntimeGetVersion(&capabilities.runtime_version), "query CUDA runtime version"); + capabilities.compute_major = properties.major; capabilities.compute_minor = properties.minor; + const auto attribute = [&](cudaDeviceAttr attr) { int value = 0; check(cudaDeviceGetAttribute(&value, attr, device), "query CUDA memory capability"); return value; }; + capabilities.unified_addressing = attribute(cudaDevAttrUnifiedAddressing); + capabilities.managed_memory = attribute(cudaDevAttrManagedMemory); + capabilities.concurrent_managed_access = attribute(cudaDevAttrConcurrentManagedAccess); + capabilities.pageable_memory_access = attribute(cudaDevAttrPageableMemoryAccess); + capabilities.host_page_tables = attribute(cudaDevAttrPageableMemoryAccessUsesHostPageTables); + capabilities.direct_managed_access = attribute(cudaDevAttrDirectManagedMemAccessFromHost); + capabilities.async_engine_count = attribute(cudaDevAttrAsyncEngineCount); + capabilities.coherent_placement_verified = 0; + check(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking), "create persistent CUDA stream"); + check(cudaEventCreate(&begin), "create CUDA timing event"); + check(cudaEventCreate(&uploaded), "create CUDA upload event"); + check(cudaEventCreate(&solved), "create CUDA solve event"); + check(cudaEventCreate(&done), "create CUDA completion event"); + initialization_ms = elapsed(started); + } + void drain() noexcept { + if (stream) cudaStreamSynchronize(stream); + in_flight = false; resident = false; + /* Read the runtime's last error before a subsequent attempted request. */ + cudaGetLastError(); + } + void reset_stats(bool replay) { + start = Clock::now(); stats = {}; + stats.initialization_ms = initialization_ms; + stats.reused_context = submissions ? 1U : 0U; + stats.resident_input = replay ? 1U : 0U; + } + void launch(U32 algorithm) { + if (algorithm != GPULAB_CUDA_DENSE && algorithm != GPULAB_CUDA_SPARSE) + throw std::runtime_error("unknown CUDA liveness algorithm"); + const U32 factor = algorithm == GPULAB_CUDA_SPARSE ? 3 : 2; + for (const auto &bucket : buckets) { + const size_t dynamic_bytes = size_t(bucket.max_blocks) * factor * sizeof(U32); + if (dynamic_bytes + 64 > properties.sharedMemPerBlock) + throw std::runtime_error("function exceeds CUDA per-block shared memory; route it to CPU"); + U32 threads = 32; + while (threads < bucket.max_blocks && threads < 256) threads *= 2; + if (algorithm == GPULAB_CUDA_DENSE) + dense<<>>( + groups.gpu() + bucket.offset, successors_offsets.gpu(), successors.gpu(), + uses.gpu(), defs.gpu(), phi.gpu(), output.gpu(), progress.gpu() + bucket.offset); + else + sparse<<>>( + groups.gpu() + bucket.offset, predecessors_offsets.gpu(), predecessors.gpu(), + uses.gpu(), defs.gpu(), phi.gpu(), output.gpu(), progress.gpu() + bucket.offset); + check(cudaGetLastError(), "launch CUDA liveness kernel"); + } + check(cudaEventRecord(solved, stream), "record CUDA solve event"); + output.download(size_t(dimensions.cell_count) * sizeof(U32), stream); + progress.download(size_t(dimensions.group_count) * sizeof(Progress), stream); + check(cudaEventRecord(done, stream), "record CUDA completion event"); + in_flight = true; ++submissions; + } +}; + +void validate(const GPULabCudaInput &in) { + if (!in.function_count || !in.block_count || !in.cell_count || !in.group_count || + !in.groups || !in.successor_offsets || !in.predecessor_offsets || !in.uses || !in.defs || !in.phi_out || + (in.edge_count && (!in.successors || !in.predecessors))) throw std::runtime_error("invalid CUDA input dimensions or pointers"); + if (in.successor_offsets[0] || in.predecessor_offsets[0] || + in.successor_offsets[in.block_count] != in.edge_count || in.predecessor_offsets[in.block_count] != in.edge_count) + throw std::runtime_error("CUDA CSR offsets do not span edges"); + for (U32 row = 0; row < in.block_count; ++row) { + if (in.successor_offsets[row] > in.successor_offsets[row + 1] || in.predecessor_offsets[row] > in.predecessor_offsets[row + 1] || + in.successor_offsets[row + 1] > in.edge_count || in.predecessor_offsets[row + 1] > in.edge_count) + throw std::runtime_error("nonmonotone CUDA CSR offsets"); + } + for (U32 e = 0; e < in.edge_count; ++e) + if (in.successors[e] >= in.block_count || in.predecessors[e] >= in.block_count) + throw std::runtime_error("CUDA CFG edge exceeds block range"); + U32 next_block = 0, next_group = 0, functions = 0, largest = 0; + uint64_t next_cell = 0; + while (next_group < in.group_count) { + const U32 g = next_group; + const auto group = in.groups[g]; + if (!group.block_count || !group.words || group.word >= group.words || + uint64_t(group.block_base) + group.block_count > in.block_count || + uint64_t(group.row_base) + uint64_t(group.block_count) * group.words > in.cell_count) + throw std::runtime_error("invalid CUDA function-word group"); + if (group.block_base != next_block || group.row_base != next_cell || group.word != 0 || + uint64_t(g) + group.words > in.group_count) + throw std::runtime_error("CUDA function-word groups must cover each cell in packed order"); + for (U32 word = 0; word < group.words; ++word) { + const auto other = in.groups[g + word]; + if (other.block_base != group.block_base || other.block_count != group.block_count || + other.row_base != group.row_base || other.words != group.words || other.word != word) + throw std::runtime_error("duplicate or inconsistent CUDA function-word group"); + } + for (U32 row = group.block_base; row < group.block_base + group.block_count; ++row) { + for (U32 e = in.successor_offsets[row]; e < in.successor_offsets[row + 1]; ++e) + if (in.successors[e] < group.block_base || in.successors[e] >= group.block_base + group.block_count) + throw std::runtime_error("CUDA successor escapes function"); + for (U32 e = in.predecessor_offsets[row]; e < in.predecessor_offsets[row + 1]; ++e) + if (in.predecessors[e] < group.block_base || in.predecessors[e] >= group.block_base + group.block_count) + throw std::runtime_error("CUDA predecessor escapes function"); + } + next_block += group.block_count; + next_cell += uint64_t(group.block_count) * group.words; + next_group += group.words; ++functions; largest = std::max(largest, group.block_count); + } + if (next_block != in.block_count || next_cell != in.cell_count || functions != in.function_count || largest != in.max_blocks) + throw std::runtime_error("CUDA function-word groups leave uncovered blocks or cells"); + std::vector> forward, reverse; + forward.reserve(in.edge_count); reverse.reserve(in.edge_count); + for (U32 row = 0; row < in.block_count; ++row) { + for (U32 e = in.successor_offsets[row]; e < in.successor_offsets[row + 1]; ++e) + forward.emplace_back(row, in.successors[e]); + for (U32 e = in.predecessor_offsets[row]; e < in.predecessor_offsets[row + 1]; ++e) + reverse.emplace_back(in.predecessors[e], row); + } + std::sort(forward.begin(), forward.end()); std::sort(reverse.begin(), reverse.end()); + if (forward != reverse) throw std::runtime_error("CUDA predecessor CSR does not reverse successor CSR"); +} +U32 bucket_key(U32 blocks) { U32 value = 1; while (value < blocks && value <= UINT32_MAX / 2) value *= 2; return value; } +} // namespace + +extern "C" uint32_t gpulab_cuda_abi_version(void) { return 1; } +extern "C" void *gpulab_cuda_create(char *err, size_t capacity) { + try { auto context = std::make_unique(); context->initialize(); return context.release(); } + catch (const std::exception &e) { error(err, capacity, e.what()); return nullptr; } +} +extern "C" int gpulab_cuda_capabilities(void *pointer, GPULabCudaCapabilities *out, char *err, size_t capacity) { + if (!pointer || !out) { error(err, capacity, "null CUDA context or capabilities output"); return -1; } + *out = static_cast(pointer)->capabilities; return 0; +} +extern "C" int gpulab_cuda_submit(void *pointer, const GPULabCudaInput *input, U32 algorithm, char *err, size_t capacity) { + if (!pointer || !input) { error(err, capacity, "null CUDA context or input"); return -1; } + auto &ctx = *static_cast(pointer); + if (ctx.in_flight) { error(err, capacity, "CUDA context already has an in-flight request"); return -1; } + try { + ctx.reset_stats(false); + if (algorithm != GPULAB_CUDA_DENSE && algorithm != GPULAB_CUDA_SPARSE) + throw std::runtime_error("unknown CUDA liveness algorithm"); + validate(*input); + const auto &in = *input; + ctx.dimensions = in; ctx.resident = false; + const size_t offsets_bytes = (size_t(in.block_count) + 1) * sizeof(U32), edge_bytes = size_t(in.edge_count) * sizeof(U32); + const size_t cell_bytes = size_t(in.cell_count) * sizeof(U32), group_bytes = size_t(in.group_count) * sizeof(GPULabCudaGroup); + ctx.groups.ensure(group_bytes, ctx.stats); + ctx.successors_offsets.ensure(offsets_bytes, ctx.stats); ctx.predecessors_offsets.ensure(offsets_bytes, ctx.stats); + ctx.successors.ensure(edge_bytes, ctx.stats); ctx.predecessors.ensure(edge_bytes, ctx.stats); + ctx.uses.ensure(cell_bytes, ctx.stats); ctx.defs.ensure(cell_bytes, ctx.stats); ctx.phi.ensure(cell_bytes, ctx.stats); + ctx.output.ensure(cell_bytes, ctx.stats); ctx.progress.ensure(size_t(in.group_count) * sizeof(Progress), ctx.stats); + ctx.groups.capture(in.groups, group_bytes); + auto *groups = ctx.groups.cpu(); + std::sort(groups, groups + in.group_count, [](const auto &a, const auto &b) { return bucket_key(a.block_count) < bucket_key(b.block_count); }); + ctx.buckets.clear(); + for (U32 g = 0; g < in.group_count;) { + U32 end = g + 1, largest = groups[g].block_count, key = bucket_key(largest); + while (end < in.group_count && bucket_key(groups[end].block_count) == key) { largest = std::max(largest, groups[end].block_count); ++end; } + ctx.buckets.push_back({g, end - g, largest}); g = end; + } + ctx.successors_offsets.capture(in.successor_offsets, offsets_bytes); ctx.predecessors_offsets.capture(in.predecessor_offsets, offsets_bytes); + ctx.successors.capture(in.successors, edge_bytes); ctx.predecessors.capture(in.predecessors, edge_bytes); + ctx.uses.capture(in.uses, cell_bytes); ctx.defs.capture(in.defs, cell_bytes); ctx.phi.capture(in.phi_out, cell_bytes); + ctx.stats.staging_ms = elapsed(ctx.start); + check(cudaEventRecord(ctx.begin, ctx.stream), "record CUDA upload start"); + ctx.groups.upload(group_bytes, ctx.stream); + ctx.successors_offsets.upload(offsets_bytes, ctx.stream); ctx.predecessors_offsets.upload(offsets_bytes, ctx.stream); + ctx.successors.upload(edge_bytes, ctx.stream); ctx.predecessors.upload(edge_bytes, ctx.stream); + ctx.uses.upload(cell_bytes, ctx.stream); ctx.defs.upload(cell_bytes, ctx.stream); ctx.phi.upload(cell_bytes, ctx.stream); + check(cudaEventRecord(ctx.uploaded, ctx.stream), "record CUDA upload completion"); + ctx.launch(algorithm); ctx.resident = true; return 0; + } catch (const std::exception &e) { ctx.drain(); error(err, capacity, e.what()); return -1; } +} +extern "C" int gpulab_cuda_submit_resident(void *pointer, U32 algorithm, char *err, size_t capacity) { + if (!pointer) { error(err, capacity, "null CUDA context"); return -1; } + auto &ctx = *static_cast(pointer); + if (ctx.in_flight) { error(err, capacity, "CUDA context already has an in-flight request"); return -1; } + if (!ctx.resident) { error(err, capacity, "no CUDA input is resident"); return -1; } + try { + ctx.reset_stats(true); + check(cudaEventRecord(ctx.begin, ctx.stream), "record resident CUDA request"); + check(cudaEventRecord(ctx.uploaded, ctx.stream), "record resident CUDA solve start"); + ctx.launch(algorithm); return 0; + } catch (const std::exception &e) { ctx.drain(); error(err, capacity, e.what()); return -1; } +} +extern "C" int gpulab_cuda_finish(void *pointer, U32 *out, size_t count, GPULabCudaStats *stats, char *err, size_t capacity) { + if (!pointer || !out || !stats) { error(err, capacity, "null CUDA context or result"); return -1; } + auto &ctx = *static_cast(pointer); + if (!ctx.in_flight) { error(err, capacity, "no CUDA request is in flight"); return -1; } + if (count < ctx.dimensions.cell_count) { error(err, capacity, "CUDA output buffer is too short"); return -1; } + try { + check(cudaEventSynchronize(ctx.done), "wait for CUDA completion event"); + float upload_ms = 0, solve_ms = 0, download_ms = 0; + check(cudaEventElapsedTime(&upload_ms, ctx.begin, ctx.uploaded), "measure CUDA upload"); + check(cudaEventElapsedTime(&solve_ms, ctx.uploaded, ctx.solved), "measure CUDA solve"); + check(cudaEventElapsedTime(&download_ms, ctx.solved, ctx.done), "measure CUDA download"); + ctx.stats.host_to_device_ms = ctx.stats.resident_input ? 0 : upload_ms; + ctx.stats.gpu_ms = solve_ms; ctx.stats.device_to_host_ms = download_ms; + for (U32 g = 0; g < ctx.dimensions.group_count; ++g) { + const auto p = ctx.progress.cpu()[g]; + if (!p.converged) throw std::runtime_error("CUDA liveness exceeded its finite monotone convergence bound"); + ctx.stats.max_sweeps = std::max(ctx.stats.max_sweeps, p.sweeps); + ctx.stats.frontier_visits += p.visits; ctx.stats.edges_examined += p.edges; ctx.stats.discovered_bits += p.bits; + } + std::memcpy(out, ctx.output.host, size_t(ctx.dimensions.cell_count) * sizeof(U32)); + ctx.stats.total_ms = elapsed(ctx.start); *stats = ctx.stats; ctx.in_flight = false; return 0; + } catch (const std::exception &e) { ctx.drain(); error(err, capacity, e.what()); return -1; } +} +extern "C" int gpulab_cuda_pending(void *pointer, U32 *out_pending, char *err, size_t capacity) { + if (!pointer || !out_pending) { error(err, capacity, "null CUDA context or pending output"); return -1; } + auto &ctx = *static_cast(pointer); + *out_pending = 0; + if (!ctx.in_flight) return 0; + const auto status = cudaEventQuery(ctx.done); + if (status == cudaErrorNotReady) { *out_pending = 1; return 0; } + if (status == cudaSuccess) return 0; + ctx.drain(); error(err, capacity, cudaGetErrorString(status)); return -1; +} +extern "C" void gpulab_cuda_destroy(void *pointer) { delete static_cast(pointer); } diff --git a/experiments/gpu-rust-compiler/native/cuda_bridge.h b/experiments/gpu-rust-compiler/native/cuda_bridge.h new file mode 100644 index 00000000000..c15f3ece278 --- /dev/null +++ b/experiments/gpu-rust-compiler/native/cuda_bridge.h @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#ifndef GPULAB_CUDA_BRIDGE_H +#define GPULAB_CUDA_BRIDGE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct GPULabCudaGroup { + uint32_t block_base, block_count, row_base, words, word; +} GPULabCudaGroup; + +typedef struct GPULabCudaInput { + uint32_t function_count, block_count, cell_count, group_count, max_blocks, edge_count; + const GPULabCudaGroup *groups; + const uint32_t *successor_offsets, *successors; + const uint32_t *predecessor_offsets, *predecessors; + const uint32_t *uses, *defs, *phi_out; +} GPULabCudaInput; + +typedef struct GPULabCudaStats { + /* initialization_ms is the context's one-time creation cost, retained on + * later requests for provenance. total_ms excludes that creation cost. */ + double initialization_ms, staging_ms, host_to_device_ms, gpu_ms, device_to_host_ms, total_ms; + uint32_t max_sweeps, reused_context, device_allocations, pinned_allocations, resident_input; + uint64_t frontier_visits, edges_examined, discovered_bits; +} GPULabCudaStats; + +typedef struct GPULabCudaCapabilities { + char device_name[256]; + char device_uuid[64]; + int32_t driver_version, runtime_version, compute_major, compute_minor; + int32_t unified_addressing, managed_memory, concurrent_managed_access; + int32_t pageable_memory_access, host_page_tables, direct_managed_access, async_engine_count; + /* Capability discovery does not establish coherent placement performance. */ + int32_t coherent_placement_verified; +} GPULabCudaCapabilities; + +enum { GPULAB_CUDA_DENSE = 0, GPULAB_CUDA_SPARSE = 1 }; + +/* ABI version 1. Callers must serialize all calls on each context; this API + * does not provide synchronization between concurrent host callers. + * The caller owns arrays until submit returns, at which point + * every input is captured in context-owned pinned memory. Groups occur in + * function order and then word order, covering each cell exactly once; both + * CSR arrays must describe the same edge multiset. One submit may be in + * flight. Finish waits for its stream event and copies a complete result. + * Destroy drains work before releasing device/pinned buffers and CUDA state. + * Errors drain outstanding work and invalidate resident replay. There is no + * CPU fallback. A rejected concurrent submit leaves existing work pending. + * Resident replay accepts no new facts: it solves the last uploaded snapshot + * again, reseeding values/frontiers, with zero host/device input transfers. + * A too-short finish output leaves the request pending. + */ +uint32_t gpulab_cuda_abi_version(void); +void *gpulab_cuda_create(char *err, size_t err_capacity); +int gpulab_cuda_capabilities(void *context, GPULabCudaCapabilities *out, + char *err, size_t err_capacity); +int gpulab_cuda_submit(void *context, const GPULabCudaInput *input, uint32_t algorithm, + char *err, size_t err_capacity); +int gpulab_cuda_submit_resident(void *context, uint32_t algorithm, + char *err, size_t err_capacity); +/* Query the completion event without waiting or consuming the result. A zero + * pending value also covers an already completed request awaiting finish. */ +int gpulab_cuda_pending(void *context, uint32_t *out_pending, + char *err, size_t err_capacity); +int gpulab_cuda_finish(void *context, uint32_t *out, size_t out_count, + GPULabCudaStats *stats, char *err, size_t err_capacity); +void gpulab_cuda_destroy(void *context); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/experiments/gpu-rust-compiler/native/metal_bridge.h b/experiments/gpu-rust-compiler/native/metal_bridge.h new file mode 100644 index 00000000000..29653915820 --- /dev/null +++ b/experiments/gpu-rust-compiler/native/metal_bridge.h @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#ifndef GPULAB_METAL_BRIDGE_H +#define GPULAB_METAL_BRIDGE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* This layout is identical to Group in src/liveness.metal. */ +typedef struct GPULabGroup { + uint32_t block_base; + uint32_t block_count; + uint32_t row_base; + uint32_t words; + uint32_t word; +} GPULabGroup; + +typedef struct GPULabInput { + uint32_t function_count; + uint32_t block_count; + uint32_t cell_count; + uint32_t group_count; + uint32_t max_blocks; + uint32_t edge_count; + const GPULabGroup *groups; + const uint32_t *successor_offsets; /* block_count + 1 entries */ + const uint32_t *successors; /* edge_count entries, may be NULL if zero */ + const uint32_t *uses; /* cell_count entries */ + const uint32_t *defs; /* cell_count entries */ + const uint32_t *phi_out; /* cell_count entries */ +} GPULabInput; + +typedef struct GPULabStats { + double gpu_ms; /* Metal command buffer GPU timestamps, not host elapsed time */ + double total_ms; /* submit entry through completed result copy */ + uint32_t max_sweeps; + uint32_t reused_pipeline; /* 0 for the first submit, 1 for subsequent submits */ + uint32_t buffer_allocations; /* buffers created or grown by this submit */ +} GPULabStats; + +/* Every API writes a NUL-terminated diagnostic if err_capacity > 0 and err != NULL. + * create returns NULL and submit/finish return -1 on error, 0 on success. + * Input and output arrays must be valid for the declared lengths. Inputs are + * copied during submit; their storage may be released as soon as submit returns. + * A context permits one in-flight submit. Finish consumes that submit, including + * when a completed GPU command fails. A too-small output leaves it pending. + * Different contexts may be used concurrently; do not destroy a context while + * another thread is calling it. Destroy waits for an outstanding GPU command. + */ +void *gpulab_create(const char *shader_path, char *err, size_t err_capacity); +int gpulab_submit(void *context, const GPULabInput *input, + char *err, size_t err_capacity); +int gpulab_finish(void *context, uint32_t *out, size_t out_count, + GPULabStats *stats, char *err, size_t err_capacity); +void gpulab_destroy(void *context); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/experiments/gpu-rust-compiler/native/metal_bridge.mm b/experiments/gpu-rust-compiler/native/metal_bridge.mm new file mode 100644 index 00000000000..0ffb91416c4 --- /dev/null +++ b/experiments/gpu-rust-compiler/native/metal_bridge.mm @@ -0,0 +1,351 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#import +#import + +#include "metal_bridge.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static_assert(sizeof(GPULabStats) == 32 && alignof(GPULabStats) == 8, "C ABI stats layout changed"); +static_assert(sizeof(GPULabGroup) == 5 * sizeof(uint32_t), "Metal Group layout changed"); +static_assert(offsetof(GPULabGroup, word) == 4 * sizeof(uint32_t), "Metal Group offsets changed"); + +namespace { +using Clock = std::chrono::steady_clock; + +struct Context { + id device; + id queue; + id library; + id pipeline; + id buffers[9] = {}; + size_t capacities[9] = {}; + id pending; + Clock::time_point started; + std::mutex mutex; + uint32_t cells = 0; + uint32_t groups = 0; + uint32_t allocations = 0; + uint32_t reused_pipeline = 0; + uint64_t submitted_batches = 0; + NSUInteger threads = 0; +}; + +void diagnostic(char *err, size_t capacity, const std::string &message) { + if (err && capacity) std::snprintf(err, capacity, "%s", message.c_str()); +} + +std::string metal_error(const char *operation, NSError *error) { + if (!error) return std::string(operation) + ": Metal returned no diagnostic"; + NSString *description = error.localizedDescription; + NSString *domain = error.domain; + return std::string(operation) + ": " + (description.UTF8String ?: "unknown error") + + " [" + (domain.UTF8String ?: "unknown domain") + ":" + + std::to_string(error.code) + "]"; +} + +bool validate(const GPULabInput *input, uint32_t &largest, std::string &error) { + if (!input) { error = "Input descriptor is NULL"; return false; } + if (!input->function_count || !input->block_count || !input->cell_count || + !input->group_count || !input->max_blocks) { + error = "GPU input must contain nonzero functions, blocks, cells, groups and max_blocks"; + return false; + } + if (!input->groups || !input->successor_offsets || !input->uses || !input->defs || + !input->phi_out || (input->edge_count && !input->successors)) { + error = "GPU input contains a NULL array with a nonzero declared length"; + return false; + } + if (input->successor_offsets[0] != 0 || + input->successor_offsets[input->block_count] != input->edge_count) { + error = "Successor offsets must begin at zero and end at edge_count"; + return false; + } + // Packed functions and their words are dense and ordered. Validate this + // once per function to prevent overlaps, uncovered output, and CFG reads + // outside the corresponding threadgroup scratch allocation. + uint64_t next_block = 0, next_row = 0, functions = 0; + largest = 0; + for (uint64_t index = 0; index < input->group_count;) { + const GPULabGroup &group = input->groups[index]; + if (!group.block_count || !group.words || group.word != 0 || + group.block_base != next_block || group.row_base != next_row || + group.words > input->group_count - index) { + error = "Invalid dense function-word group at index " + std::to_string(index); + return false; + } + const uint64_t end_block = next_block + group.block_count; + const uint64_t end_row = next_row + uint64_t(group.block_count) * group.words; + if (end_block > input->block_count || end_row > input->cell_count) { + error = "Function group dimensions exceed the declared blocks or cells"; + return false; + } + for (uint64_t word = 0; word < group.words; ++word) { + const GPULabGroup ¤t = input->groups[index + word]; + if (current.block_base != group.block_base || current.block_count != group.block_count || + current.row_base != group.row_base || current.words != group.words || current.word != word) { + error = "Function-word groups are incomplete or overlapping"; + return false; + } + } + for (uint64_t block = next_block; block < end_block; ++block) { + const uint32_t begin = input->successor_offsets[block]; + const uint32_t end = input->successor_offsets[block + 1]; + if (begin > end || end > input->edge_count) { + error = "Successor offsets are not monotonic or exceed edge_count"; + return false; + } + for (uint64_t edge = begin; edge < end; ++edge) { + const uint32_t successor = input->successors[edge]; + if (successor < next_block || successor >= end_block) { + error = "CFG successor leaves its function at block " + std::to_string(block); + return false; + } + } + } + largest = std::max(largest, group.block_count); + next_block = end_block; + next_row = end_row; + ++functions; + index += group.words; + } + if (next_block != input->block_count || next_row != input->cell_count || + functions != input->function_count || largest > input->max_blocks) { + error = "Group coverage disagrees with function_count, block_count, cell_count or max_blocks"; + return false; + } + return true; +} + +bool ensure_buffer(Context &context, NSUInteger index, size_t bytes, + const char *label, std::string &error) { + bytes = std::max(bytes, sizeof(uint32_t)); + if (bytes > context.device.maxBufferLength) { + error = std::string(label) + " exceeds Metal maxBufferLength"; + return false; + } + if (context.capacities[index] >= bytes) return true; + size_t capacity = std::max(context.capacities[index], sizeof(uint32_t)); + while (capacity < bytes && capacity <= context.device.maxBufferLength / 2) capacity *= 2; + capacity = std::max(capacity, bytes); + id buffer = [context.device newBufferWithLength:capacity options:MTLResourceStorageModeShared]; + if (!buffer || !buffer.contents) { + error = std::string("Metal buffer allocation failed for ") + label + + " (" + std::to_string(capacity) + " bytes)"; + return false; + } + buffer.label = [NSString stringWithUTF8String:label]; + context.buffers[index] = buffer; + context.capacities[index] = capacity; + ++context.allocations; + return true; +} + +int submit(Context &context, const GPULabInput *input, char *err, size_t capacity) { + std::lock_guard lock(context.mutex); + if (context.pending) { + diagnostic(err, capacity, "Only one in-flight GPU pass per context is supported; finish the pending pass first"); + return -1; + } + const auto started = Clock::now(); + std::string error; + uint32_t largest; + if (!validate(input, largest, error)) { diagnostic(err, capacity, error); return -1; } + const size_t scratch_bytes = (size_t(largest) * sizeof(uint32_t) + 15) & ~size_t(15); + const size_t static_bytes = context.pipeline.staticThreadgroupMemoryLength; + if (static_bytes > context.device.maxThreadgroupMemoryLength || + scratch_bytes > (context.device.maxThreadgroupMemoryLength - static_bytes) / 2) { + diagnostic(err, capacity, "Function with " + std::to_string(largest) + + " blocks requires " + std::to_string(2 * scratch_bytes + static_bytes) + + " bytes of threadgroup memory; GPU limit is " + + std::to_string(context.device.maxThreadgroupMemoryLength)); + return -1; + } + const size_t sizes[9] = { + size_t(input->group_count) * sizeof(GPULabGroup), + (size_t(input->block_count) + 1) * sizeof(uint32_t), + size_t(input->edge_count) * sizeof(uint32_t), + size_t(input->cell_count) * sizeof(uint32_t), + size_t(input->cell_count) * sizeof(uint32_t), + size_t(input->cell_count) * sizeof(uint32_t), + size_t(input->cell_count) * sizeof(uint32_t), + size_t(input->group_count) * sizeof(uint32_t), + size_t(input->group_count) * sizeof(uint32_t) + }; + const char *labels[9] = {"function-word groups", "successor offsets", "CFG edges", + "local uses", "local defs", "phi edge uses", "live-in result", + "fixed-point sweeps", "convergence flags"}; + context.allocations = 0; + for (NSUInteger index = 0; index < 9; ++index) { + if (!ensure_buffer(context, index, sizes[index], labels[index], error)) { + diagnostic(err, capacity, error); return -1; + } + } + const void *sources[6] = {input->groups, input->successor_offsets, input->successors, + input->uses, input->defs, input->phi_out}; + for (NSUInteger index = 0; index < 6; ++index) { + if (sizes[index]) std::memcpy(context.buffers[index].contents, sources[index], sizes[index]); + } + // Every output cell has exactly one writer. Reset status buffers so an + // incomplete dispatch cannot pass convergence validation using old data. + std::memset(context.buffers[7].contents, 0, sizes[7]); + std::memset(context.buffers[8].contents, 0, sizes[8]); + id command = [context.queue commandBuffer]; + if (!command) { diagnostic(err, capacity, "Metal command buffer creation failed"); return -1; } + id encoder = [command computeCommandEncoder]; + if (!encoder) { diagnostic(err, capacity, "Metal compute command encoder creation failed"); return -1; } + command.label = @"Persistent native Rust IR liveness"; + encoder.label = @"Batched exact bitset fixed point"; + [encoder setComputePipelineState:context.pipeline]; + for (NSUInteger index = 0; index < 9; ++index) { + [encoder setBuffer:context.buffers[index] offset:0 atIndex:index]; + } + [encoder setThreadgroupMemoryLength:scratch_bytes atIndex:0]; + [encoder setThreadgroupMemoryLength:scratch_bytes atIndex:1]; + [encoder dispatchThreadgroups:MTLSizeMake(input->group_count, 1, 1) + threadsPerThreadgroup:MTLSizeMake(context.threads, 1, 1)]; + [encoder endEncoding]; + context.started = started; + context.cells = input->cell_count; + context.groups = input->group_count; + context.reused_pipeline = context.submitted_batches ? 1 : 0; + context.pending = command; + ++context.submitted_batches; + [command commit]; + return 0; +} + +int finish(Context &context, uint32_t *out, size_t out_count, GPULabStats *stats, + char *err, size_t capacity) { + std::lock_guard lock(context.mutex); + if (!context.pending) { diagnostic(err, capacity, "No in-flight GPU pass"); return -1; } + if (!out || out_count < context.cells) { + diagnostic(err, capacity, "Output buffer must contain at least " + + std::to_string(context.cells) + " uint32_t entries; pending pass retained"); + return -1; + } + id command = context.pending; + [command waitUntilCompleted]; + context.pending = nil; + if (command.status != MTLCommandBufferStatusCompleted) { + diagnostic(err, capacity, metal_error("GPU command failed", command.error)); return -1; + } + const double gpu_ms = (command.GPUEndTime - command.GPUStartTime) * 1000.0; + if (!std::isfinite(gpu_ms) || gpu_ms <= 0.0) { + diagnostic(err, capacity, "GPU timestamps are unavailable; cannot verify GPU execution time"); + return -1; + } + const uint32_t *statuses = static_cast(context.buffers[8].contents); + const uint32_t *sweeps = static_cast(context.buffers[7].contents); + uint32_t max_sweeps = 0; + for (uint32_t index = 0; index < context.groups; ++index) { + if (statuses[index] != 1) { + diagnostic(err, capacity, "GPU fixed point did not converge for group " + std::to_string(index)); + return -1; + } + max_sweeps = std::max(max_sweeps, sweeps[index]); + } + std::memcpy(out, context.buffers[6].contents, size_t(context.cells) * sizeof(uint32_t)); + if (stats) { + stats->gpu_ms = gpu_ms; + stats->total_ms = std::chrono::duration(Clock::now() - context.started).count(); + stats->max_sweeps = max_sweeps; + stats->reused_pipeline = context.reused_pipeline; + stats->buffer_allocations = context.allocations; + } + return 0; +} +} // namespace + +extern "C" void *gpulab_create(const char *shader_path, char *err, size_t err_capacity) { + @autoreleasepool { + diagnostic(err, err_capacity, ""); + if (!shader_path || !*shader_path) { + diagnostic(err, err_capacity, "Shader path must not be empty"); return nullptr; + } + try { + NSString *path = [NSString stringWithUTF8String:shader_path]; + if (!path) { diagnostic(err, err_capacity, "Shader path is not UTF-8"); return nullptr; } + NSError *error = nil; + NSString *source = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:&error]; + if (!source) { + diagnostic(err, err_capacity, metal_error("Could not read Metal shader", error)); return nullptr; + } + id device = MTLCreateSystemDefaultDevice(); + if (!device) { diagnostic(err, err_capacity, "No Metal GPU is available"); return nullptr; } + id queue = [device newCommandQueue]; + if (!queue) { diagnostic(err, err_capacity, "Metal command queue creation failed"); return nullptr; } + id library = [device newLibraryWithSource:source options:nil error:&error]; + if (!library) { + diagnostic(err, err_capacity, metal_error("Metal shader compilation failed", error)); return nullptr; + } + id function = [library newFunctionWithName:@"liveness"]; + if (!function) { diagnostic(err, err_capacity, "Metal shader has no liveness kernel"); return nullptr; } + id pipeline = [device newComputePipelineStateWithFunction:function error:&error]; + if (!pipeline) { + diagnostic(err, err_capacity, metal_error("Metal pipeline creation failed", error)); return nullptr; + } + const NSUInteger width = pipeline.threadExecutionWidth; + const NSUInteger limit = std::min(pipeline.maxTotalThreadsPerThreadgroup, + device.maxThreadsPerThreadgroup.width); + if (!width || width > limit) { + diagnostic(err, err_capacity, "Metal pipeline cannot support one SIMD threadgroup"); return nullptr; + } + Context *context = new Context; + context->device = device; + context->queue = queue; + context->library = library; + context->pipeline = pipeline; + context->threads = std::max(width, std::min(NSUInteger(64), limit) / width * width); + return context; + } catch (const std::exception &exception) { + diagnostic(err, err_capacity, std::string("Native Metal context creation failed: ") + exception.what()); + return nullptr; + } + } +} + +extern "C" int gpulab_submit(void *context, const GPULabInput *input, + char *err, size_t err_capacity) { + @autoreleasepool { + diagnostic(err, err_capacity, ""); + if (!context) { diagnostic(err, err_capacity, "GPU context is NULL"); return -1; } + try { return submit(*static_cast(context), input, err, err_capacity); } + catch (const std::exception &exception) { + diagnostic(err, err_capacity, std::string("Native Metal submit failed: ") + exception.what()); + return -1; + } + } +} + +extern "C" int gpulab_finish(void *context, uint32_t *out, size_t out_count, + GPULabStats *stats, char *err, size_t err_capacity) { + @autoreleasepool { + diagnostic(err, err_capacity, ""); + if (stats) std::memset(stats, 0, sizeof(*stats)); + if (!context) { diagnostic(err, err_capacity, "GPU context is NULL"); return -1; } + try { return finish(*static_cast(context), out, out_count, stats, err, err_capacity); } + catch (const std::exception &exception) { + diagnostic(err, err_capacity, std::string("Native Metal finish failed: ") + exception.what()); + return -1; + } + } +} + +extern "C" void gpulab_destroy(void *context) { + @autoreleasepool { + if (!context) return; + Context *owned = static_cast(context); + if (owned->pending) [owned->pending waitUntilCompleted]; + delete owned; + } +} diff --git a/experiments/gpu-rust-compiler/scripts/benchmark_cuda.py b/experiments/gpu-rust-compiler/scripts/benchmark_cuda.py new file mode 100644 index 00000000000..ba6a5b7c908 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/benchmark_cuda.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmark native CPU and CUDA on identical synthetic liveness workpacks. + +Independent Python set results are correctness oracles, never CPU performance +baselines. Complete compilation measures only the supported scalar demo, using +fresh processes and including CUDA initialization on every compilation. +""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import math +import os +import platform +import random +import statistics +import struct +import subprocess +import sys +import time +from pathlib import Path + +from export_cuda import pack, read_result +from verify_cuda import oracle_words + +ROOT = Path(__file__).resolve().parents[1] +GENERATOR_SEED = 10091709 +ORDER_SEED = 10090031 + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def statistics_ms(samples): + values = list(samples) + if not values or any(not isinstance(value, (int, float)) or + not math.isfinite(value) or value <= 0 for value in values): + raise ValueError("Timing samples must be finite positive milliseconds") + return {"median_ms": statistics.median(values), "min_ms": min(values), + "max_ms": max(values), "samples_ms": values} + + +def synthetic_workloads(): + """Fixed seeded compact CFGs and adversarial long dependency chains.""" + rng = random.Random(GENERATOR_SEED) + + def compact(name, blocks=16, values=256, uses=4, definitions=3): + rows = [] + for index in range(blocks): + targets = ([index + 1] if index + 1 < blocks else []) + if index > 1 and rng.random() < 0.35: + targets.append(rng.randrange(index)) + rows.append({"name": str(index), "successors": sorted(set(targets)), + "use": rng.sample(range(values), min(values, uses)), + "def": rng.sample(range(values), min(values, definitions))}) + edges = [{"from": index, "to": target, + "values": rng.sample(range(values), min(values, 2))} + for index, row in enumerate(rows) for target in row["successors"]] + return {"name": name, "value_count": values, "blocks": rows, + "phi_edge_uses": edges} + + patterns = [compact(f"pattern_{index}") for index in range(16)] + + def batch(count, prefix): + # Repeated shapes model a batch of small independent functions. Sharing + # immutable Python objects only affects untimed workload generation. + return [dict(patterns[index % len(patterns)], name=f"{prefix}_{index}") + for index in range(count)] + + def chain(index): + return {"name": f"chain_{index}", "value_count": 65, + "blocks": [{"name": str(block), + "successors": [block + 1] if block < 255 else [], + "use": [0, 32, 64] if block == 255 else [], "def": []} + for block in range(256)], "phi_edge_uses": []} + + cases = [("tiny", [compact(f"tiny_{index}", blocks=5, values=65) for index in range(4)]), + ("compact-batch-64", batch(64, "compact")), + ("compact-batch-512", batch(512, "compact")), + ("compact-batch", batch(4096, "compact")), + ("few-wide", [compact(f"wide_{index}", blocks=32, values=8192, + uses=128, definitions=64) for index in range(8)]), + ("long-chain", [chain(index) for index in range(64)]), + ("mixed-compact-and-long", batch(2048, "mixed") + + [chain(index) for index in range(16)])] + return [{"version": 1, "name": name, "functions": functions} + for name, functions in cases] + + +def invoke(arguments, timeout=600): + command = [str(value) for value in arguments] + started = time.perf_counter() + completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + elapsed_ms = (time.perf_counter() - started) * 1000 + if completed.returncode: + raise RuntimeError(f"Command exited {completed.returncode}: {command[0]}\n" + f"{completed.stderr or completed.stdout}") + return completed, elapsed_ms + + +def validate_pass(metrics, backend, actual, expected, repeats): + if actual != expected: + raise ValueError(f"{backend} output differs from the independent set oracle") + if (metrics.get("backend") != backend or metrics.get("repeat_outputs_equal") is not True + or metrics.get("repeats") != repeats): + raise ValueError(f"{backend} repetition evidence is incomplete") + statistics_ms(metrics.get("warm_pass_end_to_end_ms", {}).get("samples_ms", [])) + if len(metrics["warm_pass_end_to_end_ms"]["samples_ms"]) != repeats: + raise ValueError(f"{backend} sample count differs from requested repeats") + if backend == "cuda": + if (metrics.get("converged") is not True or not metrics.get("device_name") or + not metrics.get("gpu_execution_ms", {}).get("samples_ms")): + raise ValueError("GPU execution evidence is incomplete") + statistics_ms(metrics["gpu_execution_ms"]["samples_ms"]) + + +def comparisons(cpu, cuda): + cpu_ms = cpu["warm_pass_end_to_end_ms"]["median_ms"] + resident_ms = cuda["warm_pass_end_to_end_ms"]["median_ms"] if cuda else None + update = cuda.get("warm_input_update_end_to_end_ms") if cuda else None + update_ms = update["median_ms"] if update else None + return {"cpu_selected_variant": cpu.get("selected_variant"), + "cpu_warm_median_ms": cpu_ms, + "cuda_status": "verified" if cuda else "unavailable", + "cuda_resident_input_median_ms": resident_ms, + "cuda_input_update_median_ms": update_ms, + "cpu_over_cuda_resident_input_ratio": cpu_ms / resident_ms if resident_ms else None, + "cpu_over_cuda_input_update_ratio": cpu_ms / update_ms if update_ms else None, + "ratio_definition": "CPU median divided by CUDA median; above 1 means CUDA was faster for this scope.", + "cpu_scope": "Fastest measured native serial/function-parallel/word-parallel solver; includes allocation/reset, result scatter and thread startup; excludes input file parsing.", + "resident_input_scope": "CUDA input already resident; includes launch/wait, convergence checks and device-to-host copies; excludes host-to-device input upload, context/allocations and file IO.", + "input_update_scope": "CUDA context and buffers already allocated; includes complete host-to-device input updates plus launch/wait, convergence checks and device-to-host copies; excludes file IO and context/allocations."} + + +def hardware(): + info = {"platform": platform.platform(), "machine": platform.machine(), + "python": platform.python_version(), "logical_cpu_count": os.cpu_count()} + if hasattr(os, "sched_getaffinity"): + info["cpu_affinity_count"] = len(os.sched_getaffinity(0)) + cpuinfo = Path("/proc/cpuinfo") + if cpuinfo.exists(): + info["cpu_models"] = sorted({line.split(":", 1)[1].strip() + for line in cpuinfo.read_text().splitlines() + if line.startswith("model name") and ":" in line}) + for name in ("cpu.max", "cpuset.cpus.effective"): + path = Path("/sys/fs/cgroup") / name + if path.is_file(): + info[f"cgroup_{name}"] = path.read_text().strip() + return info + + +def related_evidence(): + result = {} + for name in ("ci-provenance.json", "cuda-verification.json"): + path = ROOT / "results" / name + if not path.is_file(): + result[name] = {"status": "absent", "path": str(path)} + continue + source = json.loads(path.read_text()) + detail = {"status": "present", "path": str(path), "sha256": sha256(path)} + if name == "ci-provenance.json": + detail["provenance"] = source + else: + detail.update(verification_status=source.get("status"), + device=source.get("device"), + cuda_compiled=source.get("cuda_compiled"), + gpu_executed=source.get("gpu_executed"), + nvcc_version=source.get("nvcc_version")) + result[name] = detail + return result + + +def benchmark_passes(args, report, directory): + report["analysis_workloads"] = [] + rng = random.Random(ORDER_SEED) + for workload in synthetic_workloads(): + if args.workloads and workload["name"] not in args.workloads: + continue + path = directory / f"{workload['name']}.bin" + path.write_bytes(pack(workload)) + counts = struct.unpack_from("<6I", path.read_bytes(), 4) + record = {"name": workload["name"], + "functions": counts[0], "blocks": counts[1], "cells": counts[2], + "groups": counts[3], "max_blocks": counts[4], "edges": counts[5], + "input_sha256": sha256(path), "input_file": str(path), + "generator_seed": GENERATOR_SEED, "scope": "Synthetic liveness pass, not a Rust crate."} + if workload["name"] == "mixed-compact-and-long": + record["cuda_shared_memory_scope"] = "The existing CUDA kernel reserves shared memory using the largest function in the pack for every group; long functions can reduce compact-group occupancy." + report["analysis_workloads"].append(record) + # Oracle work happens before either native timing and is never a CPU + # performance sample. All output words are compared after each process. + expected = oracle_words(workload) + backends = [("cpu-native", args.cpu_binary)] + if not args.cpu_only: + backends.append(("cuda", args.cuda_binary)) + rng.shuffle(backends) + record["invocation_order"] = [name for name, _ in backends] + for backend, binary in backends: + output = directory / f"{workload['name']}-{backend}.result.bin" + output.unlink(missing_ok=True) + command = [binary, path, output, args.repeats] + completed, elapsed = invoke(command) + metrics = json.loads(completed.stdout) + validate_pass(metrics, backend, read_result(output), expected, args.repeats) + update = metrics.get("warm_input_update_end_to_end_ms") + if backend == "cuda" and update is None: + raise ValueError("CUDA input-update-inclusive timing evidence is absent") + if update is not None: + statistics_ms(update["samples_ms"]) + if len(update["samples_ms"]) != args.repeats: + raise ValueError("CUDA input-update sample count differs from requested repeats") + record[backend] = {"metrics": metrics, + "external_invocation_wall_ms": elapsed, + "command": [str(value) for value in command], + "output_sha256": sha256(output), + "verified_cells": len(expected), + "verified_against_independent_oracle": True, + "external_wall_scope": "One process initialization, one first pass, all repeated warm passes, binary input/output IO and JSON output; not a single compilation."} + if backend == "cuda": + report.update(gpu_executed=True, cuda_status="analysis_verified") + record["comparison"] = comparisons(record["cpu-native"]["metrics"], + record["cuda"]["metrics"] if "cuda" in record else None) + + +def benchmark_compilations(args, report, directory): + source = ROOT / "fixtures/compiler_demo.rs" + backends = ["native_cpu", "cpu_reference_bridge"] + if not args.cpu_only: + backends.append("cuda_bridge") + samples = {backend: [] for backend in backends} + rng = random.Random(ORDER_SEED + 1) + invocation_order = [] + + def compile_one(backend): + output = directory / f"demo-{backend}" + detail_path = directory / f"demo-{backend}.json" + output.unlink(missing_ok=True) + detail_path.unlink(missing_ok=True) + if backend == "native_cpu": + command = [args.compiler_binary, source, "--backend", "cpu", + "--output", output, "--report", detail_path] + else: + command = [sys.executable, ROOT / "scripts/compile.py", source, + "--backend", "cuda" if backend == "cuda_bridge" else "cpu", + "--compiler-binary", args.compiler_binary, "--cuda-binary", args.cuda_binary, + "--output", output, "--report", detail_path] + _, elapsed = invoke(command) + detail = json.loads(detail_path.read_text()) + execution, _ = invoke([output], timeout=30) + if execution.stdout != "1460\n": + raise ValueError(f"{backend} produced unexpected demo output: {execution.stdout!r}") + if backend == "cuda_bridge": + cuda = detail.get("pass_report", {}) + if (detail.get("actual_gpu_functions", 0) < 1 or cuda.get("backend") != "cuda" + or cuda.get("converged") is not True + or cuda.get("verified_against_cpu") is not True): + raise ValueError("CUDA compilation did not provide verified GPU execution evidence") + if detail.get("total_compile_ms", 0) <= 0: + raise ValueError("Complete compilation timing is absent") + return {"external_process_wall_ms": elapsed, "report": detail, + "command": [str(value) for value in command], "output_verified": "1460\n"} + + # Warm file/Clang caches once. Subsequent CUDA processes still initialize a + # fresh CUDA context each time; this is not persistent-adapter evidence. + warmup = {backend: compile_one(backend) for backend in backends} + for repetition in range(args.compile_repeats): + order = list(backends) + rng.shuffle(order) + invocation_order.append(order) + for backend in order: + sample = compile_one(backend) + sample["repetition"] = repetition + samples[backend].append(sample) + results = {} + for backend, observations in samples.items(): + stages = sorted({key for sample in observations for key, value in sample["report"].items() + if key.endswith("_ms") and isinstance(value, (int, float)) and value > 0}) + results[backend] = { + "external_process_wall_ms": statistics_ms(sample["external_process_wall_ms"] + for sample in observations), + "reported_total_compile_ms": statistics_ms(sample["report"]["total_compile_ms"] + for sample in observations), + "reported_stage_ms": {stage: statistics_ms(sample["report"][stage] + for sample in observations) for stage in stages}, + "samples": observations} + comparison = {"cuda_status": "unavailable" if args.cpu_only else "verified", + "native_cpu_over_cuda_external_wall_ratio": None, + "cpu_reference_bridge_over_cuda_external_wall_ratio": None} + if not args.cpu_only: + cuda_ms = results["cuda_bridge"]["external_process_wall_ms"]["median_ms"] + comparison.update( + native_cpu_over_cuda_external_wall_ratio=results["native_cpu"]["external_process_wall_ms"]["median_ms"] / cuda_ms, + cpu_reference_bridge_over_cuda_external_wall_ratio=results["cpu_reference_bridge"]["external_process_wall_ms"]["median_ms"] / cuda_ms) + report["complete_compilation"] = { + "scope": "Supported scalar demo only; no full Rust/Cargo, NemoClaw or Omarchy build compatibility or acceleration evidence.", + "source": str(source), "source_sha256": sha256(source), + "compile_repeats": args.compile_repeats, "order_seed": ORDER_SEED + 1, + "invocation_order": invocation_order, "untimed_warmup": warmup, + "lifecycle": "Every sample starts a fresh compiler/bridge process. CUDA starts its separate analysis process and initializes a fresh context on every compilation. Filesystem and Clang caches are warmed once.", + "external_wall_scope": "Compiler/bridge process launch through process exit, including frontend, analysis, transfers, verification, pruning, Clang code generation/linking and reporting; executable validation occurs afterward.", + "reported_total_scope": "Compiler/bridge internal timing; excludes Python interpreter or native process launch and final report serialization. Stage timing boundaries differ between native and bridge paths.", + "native_cpu_scope": "Native structured IR and bitset CPU analysis; no Python extraction or cross-backend oracle verification.", + "bridge_scope": "CPU and CUDA bridges share Python textual IR extraction, set oracle and pruning. CUDA additionally launches the CUDA process and verifies its output against that oracle. Python CPU reference is a correctness baseline, not an optimized CPU solver.", + "comparison": comparison, "results": results} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler-binary", type=Path, + default=ROOT / ".build/compiler/release/gpu-rust-compiler") + parser.add_argument("--cpu-binary", type=Path, + default=ROOT / ".build/compiler/release/gpu-cpu-liveness") + parser.add_argument("--cuda-binary", type=Path, default=ROOT / ".build/cuda/cuda-liveness") + parser.add_argument("--repeats", type=int, default=31) + parser.add_argument("--compile-repeats", type=int, default=9) + parser.add_argument("--report", type=Path, default=ROOT / "results/cuda-benchmark.json") + parser.add_argument("--cpu-only", action="store_true", + help="Validate CPU measurements locally; record CUDA unavailable without speedup claims") + parser.add_argument("--workloads", nargs="+", + choices=("tiny", "compact-batch-64", "compact-batch-512", "compact-batch", + "few-wide", "long-chain", "mixed-compact-and-long"), + help="Run a selected subset for local validation; default benchmarks all workloads") + args = parser.parse_args() + for field in ("repeats", "compile_repeats"): + if not 1 <= getattr(args, field) <= 10000: + parser.error(f"--{field.replace('_', '-')} must be between 1 and 10000") + report = {"status": "not_run", "recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "scope": "Synthetic compiler-pass and supported scalar-subset compilation benchmark only.", + "hardware": hardware(), "source_revision": os.environ.get("CANDIDATE_SHA") or os.environ.get("GITHUB_SHA"), + "generator_seed": GENERATOR_SEED, "analysis_order_seed": ORDER_SEED, + "repeats": args.repeats, "compile_repeats": args.compile_repeats, + "gpu_executed": False, "cuda_status": "unavailable" if args.cpu_only else "not_run"} + directory = ROOT / ".build/benchmarks/cuda" + directory.mkdir(parents=True, exist_ok=True) + exit_code = 1 + try: + binaries = {"compiler": args.compiler_binary, "cpu": args.cpu_binary} + if not args.cpu_only: + binaries["cuda"] = args.cuda_binary + for name, binary in binaries.items(): + if not binary.is_file(): + raise RuntimeError(f"{name} binary unavailable: {binary}") + report["binaries"] = {name: {"path": str(path.resolve()), "sha256": sha256(path)} + for name, path in binaries.items()} + implementation = [ROOT / "scripts/benchmark_cuda.py", ROOT / "scripts/compile.py", + ROOT / "scripts/verify_cuda.py", ROOT / "cuda/liveness.cu"] + implementation += sorted((ROOT / "compiler/src").rglob("*.rs")) + report["implementation_sha256"] = {str(path.relative_to(ROOT)): sha256(path) + for path in implementation} + report["related_evidence"] = related_evidence() + clang, _ = invoke(["clang", "--version"], timeout=30) + report["clang_version"] = clang.stdout.strip() + benchmark_passes(args, report, directory) + benchmark_compilations(args, report, directory) + report.update(status="cpu_only_verified" if args.cpu_only else "verified", + gpu_executed=not args.cpu_only, + cuda_status="unavailable" if args.cpu_only else "verified") + exit_code = 0 + except (OSError, RuntimeError, ValueError, subprocess.TimeoutExpired) as error: + report.update(status="failed", error=str(error)) + finally: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"report": str(args.report), "status": report["status"], + "analysis_comparisons": {record["name"]: record.get("comparison") + for record in report.get("analysis_workloads", [])}, + "complete_compilation_comparison": report.get("complete_compilation", {}).get("comparison"), + "error": report.get("error")}, indent=2)) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/gpu-rust-compiler/scripts/benchmark_native.py b/experiments/gpu-rust-compiler/scripts/benchmark_native.py new file mode 100755 index 00000000000..879df8f03c1 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/benchmark_native.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare subset compilation with fresh and persistent native workers.""" +import argparse +from datetime import datetime, timezone +import hashlib +import json +import platform +import statistics +import subprocess +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BINARY = ROOT / ".build/compiler/release/gpu-rust-compiler" + + +def command(arguments): + result = subprocess.run([str(value) for value in arguments], capture_output=True, text=True) + if result.returncode: + raise RuntimeError(result.stderr or result.stdout) + return result + + +class Worker: + def __init__(self, backend): + self.process = subprocess.Popen([str(BINARY), "--serve", "--backend", backend, "--verify"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, bufsize=1) + def compile(self, source, output, report): + fields = (source, output, report) + if any('\t' in str(value) or '\n' in str(value) for value in fields): + raise ValueError("Worker paths contain protocol delimiters") + self.process.stdin.write("compile\t" + '\t'.join(str(value) for value in fields) + "\n") + self.process.stdin.flush() + line = self.process.stdout.readline() + if not line: + raise RuntimeError("Native worker ended without a response") + value = json.loads(line) + if value.get("status") != "ok": + raise RuntimeError(value) + return value + def close(self): + if self.process.poll() is None: + self.process.stdin.write("quit\n"); self.process.stdin.flush() + self.process.communicate(timeout=20) + if self.process.returncode: + raise RuntimeError(f"Worker exited {self.process.returncode}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repeats", type=int, default=15) + parser.add_argument("--backends", nargs="+", choices=("cpu", "metal", "hybrid"), + default=["cpu", "metal", "hybrid"] if platform.system() == "Darwin" else ["cpu"]) + parser.add_argument("--report", type=Path, default=ROOT / "results/native-adapter.json") + args = parser.parse_args() + if args.repeats < 1: + raise ValueError("repeats must be positive") + # Build outside the measured interval, including the in-process Metal dylib. + command(["cargo", "build", "--locked", "--release", "--manifest-path", ROOT / "compiler/Cargo.toml", "--target-dir", ROOT / ".build/compiler"]) + directory = ROOT / ".build/native-benchmark" + directory.mkdir(parents=True, exist_ok=True) + source = ROOT / "fixtures/compiler_demo.rs" + reference = directory / "reference.rs" + reference.write_text(f'mod p{{include!("{source}");pub fn run(){{println!("{{}}",main());}}}}fn main(){{p::run();}}') + expected = "1460\n" + invocations = {"rustc_process": ["rustc", "--edition=2021", "-Awarnings", "-C", "overflow-checks=off", "-C", "opt-level=0", reference, "-o", directory / "rustc_process"]} + for mode in dict.fromkeys(args.backends): + invocations[f"native_{mode}_process"] = [BINARY, source, "--backend", mode, "--verify", "--output", directory / f"native_{mode}_process", "--report", directory / f"native_{mode}_process.json"] + workers = {f"native_{mode}_worker": Worker(mode) for mode in dict.fromkeys(args.backends)} + names = list(invocations) + list(workers) + samples = {name: [] for name in names} + details = {name: [] for name in names} + cold_workers = {} + + def compile_one(name): + if name in workers: + return workers[name].compile(source, directory / name, directory / f"{name}.json") + result = command(invocations[name]) + return json.loads(result.stdout) if name.startswith("native_") else {} + + def verify_output(name): + if command([directory / name]).stdout != expected: + raise RuntimeError(f"Native program output differs: {name}") + + try: + # Initial requests expose pipeline creation; all timed worker requests are hot. + for name in names: + start = time.perf_counter(); detail = compile_one(name) + elapsed = (time.perf_counter() - start) * 1000 + verify_output(name) + if name in workers: + cold_workers[name] = {"external_elapsed_ms": elapsed, "report": detail} + for repetition in range(args.repeats): + for offset in range(len(names)): + name = names[(repetition + offset) % len(names)] + start = time.perf_counter(); detail = compile_one(name) + samples[name].append((time.perf_counter() - start) * 1000) + details[name].append(detail) + verify_output(name) # Outside compilation timing. + if "native_metal_worker" in details: + metal_reports = details["native_metal_worker"] + if not all(report["gpu_pipeline_reused"] and report["gpu_pipeline_creations"] == 1 + and report["gpu_buffer_allocations"] == 0 and report["actual_gpu_functions"] > 0 + for report in metal_reports): + raise RuntimeError("Persistent Metal reuse was not established") + for name in workers: + if len({report["process_id"] for report in details[name]}) != 1: + raise RuntimeError("Worker process changed during measurement") + finally: + for worker in workers.values(): + worker.close() + paths = [path for folder in (ROOT / "compiler", ROOT / "native") for path in folder.rglob('*') + if path.is_file() and path.suffix in ('.rs', '.toml', '.mm', '.h') and 'target' not in path.parts] + report = { + "scope": "Complete supported-subset compilation; not full Rust language equivalence or NemoClaw acceleration.", + "recorded_at_utc": datetime.now(timezone.utc).isoformat(), "repeats": args.repeats, + "hardware": platform.platform(), "rustc": command(["rustc", "--version"]).stdout.strip(), + "clang": command(["clang", "--version"]).stdout.strip(), + "source": str(source), "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "implementation_sha256": {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(paths)}, + "measurement": "External process elapsed or persistent write-request-to-read-response time, including codegen/linking/reporting, excluding build setup and executing output. Persistent-worker initialization is reported separately. Verification is enabled for native paths.", + "remaining_codegen_subprocess": "clang", "all_native_outputs_verified": True, + "cold_worker_initial_requests": cold_workers, + "results": {name: {"median_ms": statistics.median(values), "samples_ms": values, "reports": details[name], + "command": [str(value) for value in invocations[name]] if name in invocations else [str(BINARY), "--serve", "--backend", name.split('_')[1], "--verify"]} + for name, values in samples.items()}, + } + destination = args.report + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"report": str(destination), "medians_ms": {name: round(result["median_ms"], 3) for name, result in report["results"].items()}}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/gpu-rust-compiler/scripts/build_native.sh b/experiments/gpu-rust-compiler/scripts/build_native.sh new file mode 100755 index 00000000000..927635a2155 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/build_native.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu +lab_dir=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cargo build --locked --release --manifest-path "$lab_dir/compiler/Cargo.toml" --target-dir "$lab_dir/.build/compiler" +printf 'Built %s\n' "$lab_dir/.build/compiler/release/gpu-rust-compiler" diff --git a/experiments/gpu-rust-compiler/scripts/compile.py b/experiments/gpu-rust-compiler/scripts/compile.py new file mode 100755 index 00000000000..fa95821f761 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/compile.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compile the supported Rust subset through a portable CPU or CUDA bridge. + +CPU uses the independent Python set oracle. CUDA uses a separate native process; +it is not the native compiler's persistent Metal adapter. +""" +import argparse +import hashlib +import json +import subprocess +import time +from pathlib import Path +from prepare_workload import parse_llvm_ir +from export_cuda import pack, read_result +from optimize_ir import optimize +from liveness_reference import solve_reference + +ROOT = Path(__file__).resolve().parents[1] + + +def needs_rebuild(binary, sources): + return not binary.exists() or any(path.stat().st_mtime_ns > binary.stat().st_mtime_ns for path in sources) + + +def invoke(command): + completed = subprocess.run([str(part) for part in command], capture_output=True, text=True) + if completed.returncode: + raise SystemExit(completed.stderr.strip() or completed.stdout.strip() or f"Command failed: {command[0]}") + return completed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("--backend", choices=("cpu", "cuda"), default="cpu") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--compiler-binary", type=Path, + help="Use an existing native frontend without rebuilding; must match this source revision") + parser.add_argument("--cuda-binary", type=Path, default=ROOT / ".build/cuda/cuda-liveness") + parser.add_argument("--report", type=Path) + args = parser.parse_args() + compiler = args.compiler_binary or ROOT / ".build/compiler/release/gpu-rust-compiler" + sources = list((ROOT / "compiler/src").glob("*.rs")) + sources += [ROOT / "compiler/Cargo.toml", ROOT / "compiler/Cargo.lock", ROOT / "compiler/build.rs"] + sources += list((ROOT / "native").glob("*")) + if args.compiler_binary is not None: + compiler = compiler.resolve() + if not compiler.is_file(): + raise SystemExit("Explicit compiler binary is unavailable") + elif needs_rebuild(compiler, sources): + invoke(["cargo", "build", "--locked", "--release", "--manifest-path", ROOT / "compiler/Cargo.toml", "--target-dir", ROOT / ".build/compiler"]) + source = args.source.resolve() + workdir = ROOT / ".build/compilations" / hashlib.sha256(str(source).encode()).hexdigest()[:12] + workdir.mkdir(parents=True, exist_ok=True) + raw_ir, final_ir = workdir / "raw.ll", workdir / "optimized.ll" + pack_path, solution_path, pass_report = workdir / "workload.json", workdir / "solution.json", workdir / "pass.json" + start = time.perf_counter() + stage_start = start + invoke([compiler, source, "--emit-llvm", raw_ir]) + frontend_ms = (time.perf_counter() - stage_start) * 1000 + stage_start = time.perf_counter() + workload = parse_llvm_ir(raw_ir.read_text(), source.name) + pack_path.write_text(json.dumps(workload)) + extraction_ms = (time.perf_counter() - stage_start) * 1000 + stage_start = time.perf_counter() + reference = solve_reference(workload) + if args.backend == "cuda": + if not args.cuda_binary.exists(): + raise SystemExit("CUDA backend binary unavailable. Run scripts/verify_cuda.py on an NVIDIA GPU runner first. CUDA was not executed.") + binary_input, binary_output = workdir / "cuda-input.bin", workdir / "cuda-output.bin" + binary_input.write_bytes(pack(workload)) + completed = invoke([args.cuda_binary, binary_input, binary_output, "1"]) + values = read_result(binary_output) + if values != reference["live_in_words"]: + raise ValueError("CUDA result differs from CPU reference") + metrics = json.loads(completed.stdout) + if (metrics.get("backend") != "cuda" or metrics.get("converged") is not True or + metrics.get("repeat_outputs_equal") is not True or + metrics.get("gpu_execution_ms", {}).get("median_ms", 0) <= 0): + raise ValueError("CUDA execution evidence is incomplete") + metrics["verified_against_cpu"] = True + reference["live_in_words"] = values + reference["requested_backend"] = "cuda" + reference["actual_gpu_functions"] = len(workload["functions"]) + solution_path.write_text(json.dumps(reference)) + pass_report.write_text(json.dumps(metrics)) + else: + solution_path.write_text(json.dumps(reference)) + pass_report.write_text(json.dumps({"backend": "cpu-reference", "gpu_executed": False})) + analysis_ms = (time.perf_counter() - stage_start) * 1000 + stage_start = time.perf_counter() + solution = json.loads(solution_path.read_text()) + optimized, removed = optimize(raw_ir.read_text(), workload, solution) + final_ir.write_text(optimized) + optimization_ms = (time.perf_counter() - stage_start) * 1000 + args.output = args.output.resolve() + args.output.parent.mkdir(parents=True, exist_ok=True) + stage_start = time.perf_counter() + invoke(["clang", "-Wno-override-module", "-O0", final_ir, "-o", args.output]) + machine_code_ms = (time.perf_counter() - stage_start) * 1000 + report = {"compiler": "new scalar Rust-subset frontend, not rustc", "backend": args.backend, + "actual_gpu_functions": solution["actual_gpu_functions"] if "actual_gpu_functions" in solution else 0, + "source": str(source), "source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "output": str(args.output), "total_compile_ms": (time.perf_counter() - start) * 1000, + "frontend_and_ir_ms": frontend_ms, "ir_extraction_ms": extraction_ms, + "analysis_with_cpu_verification_including_startup_ms": analysis_ms, "dead_code_pruning_ms": optimization_ms, + "native_codegen_and_link_ms": machine_code_ms, "dead_scalar_instructions_removed": removed, + "pass_report": json.loads(pass_report.read_text()), + "scope": "Supported scalar subset only; verified liveness feeds dead scalar code pruning; no full Rust compatibility."} + report_path = args.report or args.output.with_suffix(".compile.json") + report_path.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/gpu-rust-compiler/scripts/export_cuda.py b/experiments/gpu-rust-compiler/scripts/export_cuda.py new file mode 100755 index 00000000000..5524f23dc50 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/export_cuda.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Export the backend-neutral liveness pack to the CUDA CLI's GLC1 format.""" +import argparse +import json +import struct +from pathlib import Path + + +def pack(workload): + if workload.get("version") != 1: + raise ValueError("Unsupported workload version") + groups, successor_offsets, successors = [], [0], [] + uses, defs, phi = [], [], [] + block_base = 0 + maximum = 0 + for function in workload["functions"]: + count = len(function["blocks"]) + if count == 0 or function["value_count"] < 0: + raise ValueError("Invalid function dimensions") + width = max(1, (function["value_count"] + 31) // 32) + row_base = len(uses) + maximum = max(maximum, count) + groups.extend((block_base, count, row_base, width, word) for word in range(width)) + for block in function["blocks"]: + row_use, row_def = [0] * width, [0] * width + for key, row in (("use", row_use), ("def", row_def)): + for value in block[key]: + if not 0 <= value < function["value_count"]: + raise ValueError("Invalid local value") + row[value // 32] |= 1 << (value % 32) + uses.extend(row_use); defs.extend(row_def); phi.extend([0] * width) + for target in block["successors"]: + if not 0 <= target < count: + raise ValueError("Invalid CFG successor") + successors.append(block_base + target) + successor_offsets.append(len(successors)) + for edge in function["phi_edge_uses"]: + if not 0 <= edge["from"] < count or edge["to"] not in function["blocks"][edge["from"]]["successors"]: + raise ValueError("Invalid phi edge") + for value in edge["values"]: + if not 0 <= value < function["value_count"]: + raise ValueError("Invalid phi value") + phi[row_base + edge["from"] * width + value // 32] |= 1 << (value % 32) + block_base += count + integers = [len(workload["functions"]), block_base, len(uses), len(groups), maximum, len(successors)] + integers.extend(value for group in groups for value in group) + integers += successor_offsets + successors + uses + defs + phi + return b"GLC1" + struct.pack(f"<{len(integers)}I", *integers) + + +def read_result(path): + data = Path(path).read_bytes() + if len(data) < 8 or data[:4] != b"GLR1": + raise ValueError("Invalid CUDA result header") + cells, = struct.unpack_from(" 100_000_000: + raise ValueError("Workload exceeds 100 million bitset words") + successors, uses, definitions = [], [], [] + for block_index, block_value in enumerate(blocks): + block_label = f"{label}.blocks[{block_index}]" + block = _object(block_value, block_label) + successors.append(_ids(_field(block, "successors", block_label), len(blocks), + f"{block_label}.successors")) + uses.append(_ids(_field(block, "use", block_label), value_count, f"{block_label}.use")) + definitions.append(_ids(_field(block, "def", block_label), value_count, f"{block_label}.def")) + outgoing_phi = [set() for _ in blocks] + edges = _array(_field(function, "phi_edge_uses", label), f"{label}.phi_edge_uses") + for edge_index, edge_value in enumerate(edges): + edge_label = f"{label}.phi_edge_uses[{edge_index}]" + edge = _object(edge_value, edge_label) + source = _integer(_field(edge, "from", edge_label), f"{edge_label}.from") + target = _integer(_field(edge, "to", edge_label), f"{edge_label}.to") + if not 0 <= source < len(blocks) or not 0 <= target < len(blocks) or target not in successors[source]: + raise ValueError(f"Invalid phi edge in {label}") + outgoing_phi[source].update(_ids(_field(edge, "values", edge_label), value_count, + f"{edge_label}.values")) + validated.append((width, successors, uses, definitions, outgoing_phi)) + return validated + + +def solve_reference(workload): + """Validate input and return CPU-reference words with native-compatible offsets. + + The synchronous set algorithm is deliberately independent of the CUDA kernel + and native CPU bitsets. Validation completes before solving any function. + """ + functions = _validate(workload) + words, row_offsets, function_offsets = [], [], [] + block_base = 0 + for width, successors, uses, definitions, outgoing_phi in functions: + function_offsets.append(block_base) + live = [set() for _ in successors] + for _ in range(len(successors) + 1): + updated = [] + for block, targets in enumerate(successors): + out = outgoing_phi[block].union(*(live[target] for target in targets)) + updated.append(uses[block] | (out - definitions[block])) + if updated == live: + break + live = updated + else: + raise ValueError("CPU reference fixed point did not converge") + for values in live: + row_offsets.append(len(words)) + row = [0] * width + for value in values: + row[value // 32] |= 1 << (value % 32) + words.extend(row) + block_base += len(successors) + row_offsets.append(len(words)) + function_offsets.append(block_base) + return {"live_in_words": words, "row_offsets": row_offsets, + "function_block_offsets": function_offsets, + "requested_backend": "cpu-reference", "actual_gpu_functions": 0} diff --git a/experiments/gpu-rust-compiler/scripts/optimize_ir.py b/experiments/gpu-rust-compiler/scripts/optimize_ir.py new file mode 100644 index 00000000000..78d610cce28 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/optimize_ir.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Use solved liveness to prune dead scalar SSA operations in our compiler IR. + +Calls, loads, stores, phi nodes, and control flow are retained. This bounded pass +is for the new frontend's named-register LLVM output, not arbitrary LLVM files. +""" +import re +from prepare_workload import _TOKEN, _LABEL, _name, _opcode, _strip_comment + +PURE = set("add sub mul and or xor shl lshr ashr icmp select trunc zext sext bitcast ptrtoint inttoptr sdiv srem".split()) + + +def optimize(source, workload, solution): + lines = source.splitlines(keepends=True) + removed = set() + function_index = -1 + block_index = -1 + instructions = [] + function = None + + def finish_block(): + nonlocal instructions + if not instructions: + return + blocks = function["blocks"] + block = blocks[block_index] + names = function["value_names"] + ids = {name: i for i, name in enumerate(names)} + first = solution["function_block_offsets"][function_index] + width = max(1, (function["value_count"] + 31) // 32) + live = set() + for target in block["successors"]: + row = solution["row_offsets"][first + target] + for value, name in enumerate(names): + if solution["live_in_words"][row + value // 32] & (1 << (value % 32)): + live.add(name) + for edge in function["phi_edge_uses"]: + if edge["from"] == block_index: + live.update(names[value] for value in edge["values"]) + for line_index, tokens in reversed(instructions): + opcode, operands, definition = _opcode(tokens) + if opcode in PURE and definition is not None and definition not in live: + if definition.isdecimal(): + raise ValueError("DCE requires named registers from this compiler") + removed.add(line_index) + continue + if definition is not None: + live.discard(definition) + if opcode != "phi": + live.update(_name(token) for token in operands if token.startswith('%') and _name(token) in ids) + instructions = [] + + for line_index, original in enumerate(lines): + line = _strip_comment(original) + if line.startswith("define "): + function_index += 1 + function = workload["functions"][function_index] + if "value_names" not in function: + raise ValueError("Workload lacks SSA value names") + block_index = -1 + elif function is not None and line == "}": + finish_block() + function = None + elif function is not None and _LABEL.fullmatch(line): + finish_block() + block_index += 1 + elif function is not None and line: + if block_index < 0: + raise ValueError("DCE expects explicitly named blocks from this compiler") + instructions.append((line_index, _TOKEN.findall(line))) + if function is not None: + raise ValueError("Unterminated compiler IR") + return ''.join(line for i, line in enumerate(lines) if i not in removed), len(removed) diff --git a/experiments/gpu-rust-compiler/scripts/prepare_workload.py b/experiments/gpu-rust-compiler/scripts/prepare_workload.py new file mode 100755 index 00000000000..889c3ed5cd1 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/prepare_workload.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract an explicit SSA block-liveness workload from textual LLVM IR. + +This is a deliberately bounded parser, not an LLVM implementation. Unsupported +opcodes, unresolved locals, missing terminators and ambiguous names fail closed. +Phi operands are edge uses; folding them into GEN must apply predecessor DEF. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +from pathlib import Path + + +class IRParseError(ValueError): + pass + + +_NAME = r'(?:"(?:[^"\\]|\\.)*"|[-a-zA-Z$._0-9]+)' +_LOCAL = re.compile(r'%' + _NAME) +_TOKEN = re.compile(r'[%@]' + _NAME + r'|"(?:[^"\\]|\\.)*"|[-a-zA-Z$._0-9]+|[^\s]') +_LABEL = re.compile(r'^(' + _NAME + r')\s*:$') +_TYPE = re.compile(r'^%(' + _NAME + r')\s*=\s*type\b') +_OPS = set("""add fadd sub fsub mul fmul udiv sdiv fdiv urem srem frem + shl lshr ashr and or xor fneg icmp fcmp select freeze trunc zext sext + fptrunc fpext fptoui fptosi uitofp sitofp ptrtoint inttoptr bitcast + addrspacecast extractelement insertelement shufflevector extractvalue + insertvalue alloca load store fence cmpxchg atomicrmw getelementptr + phi call invoke callbr va_arg landingpad catchpad cleanuppad catchswitch + ret br switch indirectbr resume catchret cleanupret unreachable""".split()) +_TERMINATORS = set("ret br switch indirectbr invoke callbr resume catchret cleanupret catchswitch unreachable".split()) + + +def _name(token: str) -> str: + token = token.lstrip('%@') + if token.startswith('"'): + token = token[1:-1] + token = re.sub(r'\\([0-9A-Fa-f]{2})', lambda m: chr(int(m[1], 16)), token) + return token + + +def _strip_comment(line: str) -> str: + quoted = False + escaped = False + for i, ch in enumerate(line): + if quoted and ch == "\\" and not escaped: + escaped = True + continue + if ch == '"' and not escaped: + quoted = not quoted + if ch == ';' and not quoted: + return line[:i].strip() + escaped = False + return line.strip() + + +def _balance(text: str) -> int: + return sum(t in ('[', '(', '{') for t in _TOKEN.findall(text)) - sum( + t in (']', ')', '}') for t in _TOKEN.findall(text) + ) + + +def _split_top(tokens: list[str], separator: str = ',') -> list[list[str]]: + result, current, depth = [], [], 0 + for token in tokens: + if token == separator and depth == 0: + result.append(current) + current = [] + continue + current.append(token) + depth += token in ('[', '(', '{', '<') + depth -= token in (']', ')', '}', '>') + result.append(current) + return result + + +def _opcode(tokens: list[str]) -> tuple[str, list[str], str | None]: + definition = None + if len(tokens) > 2 and tokens[0].startswith('%') and tokens[1] == '=': + definition, tokens = _name(tokens[0]), tokens[2:] + if tokens and tokens[0] in ('tail', 'musttail', 'notail'): + tokens = tokens[1:] + if not tokens or tokens[0] not in _OPS: + raise IRParseError(f"unsupported instruction: {' '.join(tokens[:10])}") + return tokens[0], tokens[1:], definition + + +def _function_header(header: str, named_types: set[str]) -> tuple[str, list[str], str]: + tokens = _TOKEN.findall(header) + try: + gi = next(i for i, t in enumerate(tokens) if t.startswith('@')) + except StopIteration as exc: + raise IRParseError('function definition has no global name') from exc + if tokens[gi + 1] != '(': + raise IRParseError('function arguments must follow its name') + depth, end = 1, gi + 2 + while end < len(tokens) and depth: + depth += tokens[end] == '(' + depth -= tokens[end] == ')' + end += 1 + if depth: + raise IRParseError('unterminated function arguments') + args, next_unnamed = [], 0 + for arg in _split_top(tokens[gi + 2:end - 1]): + if not arg or arg == ['...']: + continue + locals_ = [t for t in arg if t.startswith('%')] + # LLVM printers name arguments; support omitted argument names as well. + if locals_ and _name(locals_[-1]) not in named_types: + name = _name(locals_[-1]) + elif locals_ and len(locals_) > 1: + raise IRParseError('argument name collides with a named LLVM type') + else: + name = str(next_unnamed) + if name.isdecimal(): + next_unnamed = max(next_unnamed, int(name) + 1) + args.append(name) + return _name(tokens[gi]), args, str(next_unnamed) + + +def _parse_function(header: str, lines: list[str], named_types: set[str]) -> dict: + fname, args, implicit_entry = _function_header(header, named_types) + blocks: list[dict] = [] + pending = '' + + def finish_instruction() -> None: + nonlocal pending + if pending: + if not blocks: + blocks.append({'name': implicit_entry, 'instructions': []}) + # Debug records do not represent executable operand uses. + if not pending.startswith('#dbg_'): + blocks[-1]['instructions'].append(_TOKEN.findall(pending)) + pending = '' + + for line in lines: + label = _LABEL.fullmatch(line) + if label: + finish_instruction() + blocks.append({'name': _name(label[1]), 'instructions': []}) + continue + continuation = line == 'cleanup' or line.startswith(('to label ', 'unwind label ', 'catch ', 'filter ')) + if pending and (_balance(pending) > 0 or continuation): + pending += ' ' + line + else: + finish_instruction() + pending = line + finish_instruction() + if not blocks: + raise IRParseError(f'{fname}: function body is empty') + block_ids = {b['name']: i for i, b in enumerate(blocks)} + if len(block_ids) != len(blocks): + raise IRParseError(f'{fname}: duplicate block name') + values: dict[str, int] = {} + for arg in args: + if arg in values: + raise IRParseError(f'{fname}: duplicate argument {arg}') + values[arg] = len(values) + parsed = [] + for block in blocks: + instructions = [_opcode(t) for t in block['instructions']] + if not instructions or instructions[-1][0] not in _TERMINATORS: + raise IRParseError(f"{fname}/{block['name']}: missing terminator") + if any(op in _TERMINATORS for op, _, _ in instructions[:-1]): + raise IRParseError(f"{fname}/{block['name']}: instruction after terminator") + for _, _, definition in instructions: + if definition is not None: + if definition in values or definition in block_ids: + raise IRParseError(f'{fname}: duplicate local definition {definition}') + values[definition] = len(values) + parsed.append(instructions) + if set(values) & named_types: + raise IRParseError(f'{fname}: local value collides with named LLVM type') + + def local_uses(tokens: list[str], labels: bool = True) -> set[int]: + uses = set() + for i, token in enumerate(tokens): + if not token.startswith('%'): + continue + name = _name(token) + if name in named_types or (labels and i and tokens[i - 1] == 'label'): + continue + if name not in values: + raise IRParseError(f'{fname}: unresolved local value %{name}') + uses.add(values[name]) + return uses + + edge_uses: dict[tuple[int, int], set[int]] = {} + result_blocks = [] + for bi, (block, instructions) in enumerate(zip(blocks, parsed)): + defs, uses, successors = set(), set(), [] + seen_non_phi = False + for op, operands, definition in instructions: + if op == 'phi': + if seen_non_phi or definition is None: + raise IRParseError(f"{fname}/{block['name']}: invalid phi placement") + # Top-level incoming pairs, including array/vector constant operands. + incoming = [part for part in _split_top(operands) if part and part[-1] == ']'] + if not incoming: + raise IRParseError(f'{fname}: phi has no incoming pairs') + for part in incoming: + # The type itself can be an array, so match the final pair + # backwards rather than assuming the first '[' starts it. + depth, start = 0, None + for j in range(len(part) - 1, -1, -1): + depth += part[j] == ']' + depth -= part[j] == '[' + if depth == 0: + start = j + break + if start is None or part[start] != '[': + raise IRParseError(f'{fname}: malformed phi incoming brackets') + pair = _split_top(part[start + 1:-1]) + if len(pair) != 2 or len(pair[1]) != 1 or not pair[1][0].startswith('%'): + raise IRParseError(f'{fname}: unsupported phi incoming pair') + pred = _name(pair[1][0]) + if pred not in block_ids: + raise IRParseError(f'{fname}: phi references unknown predecessor {pred}') + edge_uses.setdefault((block_ids[pred], bi), set()).update(local_uses(pair[0])) + else: + seen_non_phi = True + is_debug_intrinsic = op == 'call' and any(t.startswith('@llvm.dbg.') for t in operands) + if not is_debug_intrinsic: + uses.update(local_uses(operands) - defs) + if definition is not None: + defs.add(values[definition]) + if op in _TERMINATORS: + for i, token in enumerate(operands[:-1]): + if token == 'label': + target = _name(operands[i + 1]) + if target not in block_ids: + raise IRParseError(f'{fname}: unknown CFG target {target}') + if block_ids[target] not in successors: + successors.append(block_ids[target]) + if op == 'br' and len(successors) not in (1, 2): + raise IRParseError(f'{fname}: unsupported branch CFG') + if op in ('switch', 'invoke', 'callbr', 'indirectbr', 'catchret') and not successors: + raise IRParseError(f'{fname}: terminator has no CFG targets') + result_blocks.append({'name': block['name'], 'successors': successors, + 'use': sorted(uses), 'def': sorted(defs)}) + for (pred, succ) in edge_uses: + if succ not in result_blocks[pred]['successors']: + raise IRParseError(f'{fname}: phi incoming block is not a CFG predecessor') + return {'name': fname, 'blocks': result_blocks, 'value_count': len(values), 'value_names': list(values), + 'phi_edge_uses': [{'from': pred, 'to': succ, 'values': sorted(v)} + for (pred, succ), v in sorted(edge_uses.items())]} + + +def parse_llvm_ir(source: str, name: str = 'llvm-ir') -> dict: + lines = [_strip_comment(line) for line in source.splitlines()] + lines = [line for line in lines if line] + named_types = {_name(m[1]) for line in lines if (m := _TYPE.match(line))} + functions, i = [], 0 + while i < len(lines): + if not lines[i].startswith('define '): + i += 1 + continue + header = lines[i] + while not header.endswith('{'): + i += 1 + if i >= len(lines): + raise IRParseError('unterminated function header') + header += ' ' + lines[i] + i += 1 + body = [] + while i < len(lines) and lines[i] != '}': + body.append(lines[i]) + i += 1 + if i == len(lines): + raise IRParseError('unterminated function body') + try: + functions.append(_parse_function(header, body, named_types)) + except IRParseError as exc: + raise IRParseError(f'function starting with {header[:100]}: {exc}') from exc + i += 1 + if not functions: + raise IRParseError('no defined functions found in LLVM IR') + return {'version': 1, 'name': name, 'functions': functions} + + +def fold_phi_edge_uses(function: dict) -> dict: + """Make GEN suitable for IN=GEN | (union successor IN & ~DEF). + + Keep explicit edge uses for constructing OUT or checking edge semantics. + A phi operand defined in the predecessor is live out but never live in. + """ + folded = copy.deepcopy(function) + for edge in function['phi_edge_uses']: + block = folded['blocks'][edge['from']] + block['use'] = sorted(set(block['use']) | (set(edge['values']) - set(block['def']))) + return folded + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('inputs', nargs='+', type=Path) + parser.add_argument('--output', '-o', required=True, type=Path) + parser.add_argument('--name', default='rustc-llvm-ir') + args = parser.parse_args() + workload = {'version': 1, 'name': args.name, 'functions': []} + try: + for path in args.inputs: + workload['functions'].extend(parse_llvm_ir(path.read_text(), str(path))['functions']) + except (IRParseError, OSError) as exc: + parser.exit(2, f'workload extraction failed: {exc}\n') + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(workload, indent=2) + '\n') + print(f"Extracted {len(workload['functions'])} functions into {args.output}") + + +if __name__ == '__main__': + main() diff --git a/experiments/gpu-rust-compiler/scripts/verify_cuda.py b/experiments/gpu-rust-compiler/scripts/verify_cuda.py new file mode 100644 index 00000000000..bff1f518933 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/verify_cuda.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify the CUDA pass on a real CUDA device; retain explicit blocked evidence. + +The host format checker can run without CUDA. It excludes all CUDA code and is +never CUDA compilation, GPU execution, or end-to-end Rust compilation evidence. +Exit codes: 0 verified CUDA pass, 1 verification failure, 3 CUDA unavailable. +""" +import argparse +import hashlib +import json +import platform +import random +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +from export_cuda import pack, read_result + +ROOT = Path(__file__).resolve().parents[1] + +DEVICE_PROBE = r''' +#include +#include +#include +int main() { + int count = 0; + auto error = cudaGetDeviceCount(&count); + if (error != cudaSuccess || count == 0) { + std::cerr << "cudaGetDeviceCount: " << cudaGetErrorString(error) + << "; device_count=" << count << '\n'; + return 3; + } + cudaDeviceProp device{}; + error = cudaGetDeviceProperties(&device, 0); + if (error != cudaSuccess) { std::cerr << cudaGetErrorString(error); return 1; } + char pci[32]{}; + error = cudaDeviceGetPCIBusId(pci, sizeof(pci), 0); + if (error != cudaSuccess) { std::cerr << cudaGetErrorString(error); return 1; } + int driver = 0, runtime = 0; + if (cudaDriverGetVersion(&driver) != cudaSuccess || + cudaRuntimeGetVersion(&runtime) != cudaSuccess) return 1; + std::cout << "{\"device_count\":" << count << ",\"device_ordinal\":0,\"name\":" + << std::quoted(device.name) << ",\"pci_bus_id\":" << std::quoted(pci) + << ",\"compute_major\":" << device.major << ",\"compute_minor\":" << device.minor + << ",\"driver_version\":" << driver << ",\"runtime_version\":" << runtime + << ",\"global_memory_bytes\":" << device.totalGlobalMem + << ",\"uuid_hex\":\""; + for (unsigned char byte : device.uuid.bytes) + std::cout << std::hex << std::setw(2) << std::setfill('0') << unsigned(byte); + std::cout << "\"}\n"; +} +''' + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def run(command, timeout=120): + try: + done = subprocess.run([str(value) for value in command], capture_output=True, + text=True, timeout=timeout) + return {"command": [str(value) for value in command], "exit_code": done.returncode, + "stdout": done.stdout, "stderr": done.stderr} + except (OSError, subprocess.TimeoutExpired) as error: + return {"command": [str(value) for value in command], "exit_code": None, + "stdout": "", "stderr": str(error)} + + +def require_success(result): + if result["exit_code"] != 0: + raise RuntimeError(result["stderr"] or result["stdout"] or "command failed") + + +def tool_identity(name): + path = shutil.which(name) + return {"path": path, "resolved_path": str(Path(path).resolve()) if path else None, + "sha256": sha256(path) if path else None} + + +def oracle_words(workload): + """Independent set fixed point, without backend bitsets or group layouts.""" + result = [] + for function in workload["functions"]: + blocks = function["blocks"] + phi = [set() for _ in blocks] + for edge in function["phi_edge_uses"]: + phi[edge["from"]].update(edge["values"]) + live = [set() for _ in blocks] + for _ in range(len(blocks) + 2): + following = [] + for index, block in enumerate(blocks): + out = phi[index].union(*(live[target] for target in block["successors"])) + following.append(set(block["use"]) | (out - set(block["def"]))) + if following == live: + break + live = following + else: + raise RuntimeError("independent set oracle did not converge") + width = max(1, (function["value_count"] + 31) // 32) + for values in live: + row = [0] * width + for value in values: + row[value // 32] |= 1 << (value % 32) + result.extend(row) + return result + + +def workloads(): + rng = random.Random(1709) + functions = [] + for values in (0, 1, 31, 32, 33, 65, 129): + for count in (1, 7, 35): + blocks = [] + for index in range(count): + blocks.append({"name": str(index), + "successors": sorted({rng.randrange(count) for _ in range(rng.randrange(4))}), + "use": rng.sample(range(values), min(values, 4)), + "def": rng.sample(range(values), min(values, 3))}) + edges = [{"from": index, "to": target, + "values": rng.sample(range(values), min(values, 2))} + for index, block in enumerate(blocks) for target in block["successors"]] + functions.append({"name": f"seeded_{values}_{count}", "value_count": values, + "blocks": blocks, "phi_edge_uses": edges}) + path = {"name": "long_path_cycle", "value_count": 65, + "blocks": [{"name": str(index), "successors": [index + 1] if index < 255 else [128], + "use": [32, 64] if index == 255 else [], "def": []} + for index in range(256)], "phi_edge_uses": []} + functions.append(path) + small = {"name": "small", "value_count": 33, + "blocks": [{"name": "entry", "successors": [1], "use": [], "def": [0]}, + {"name": "exit", "successors": [], "use": [0, 32], "def": []}], + "phi_edge_uses": [{"from": 0, "to": 1, "values": [32]}]} + mixed = [dict(small, name=f"small_{index}") for index in range(1024)] + mixed += [dict(path, name=f"path_{index}") for index in range(4)] + return [{"version": 1, "name": "seeded-correctness", "functions": functions}, + {"version": 1, "name": "mixed-small-and-long", "functions": mixed}] + + +def host_format_check(report, directory, packs): + compiler = report["tools"]["clang++"]["path"] or report["tools"]["g++"]["path"] + if compiler is None: + report["host_format_check"] = {"status": "unavailable", "gpu_executed": False} + return + binary = directory / "cuda-format-check" + build = run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror", "-pedantic", + "-DGLC_FORMAT_CHECK_ONLY", "-x", "c++", ROOT / "cuda/liveness.cu", "-o", binary]) + report["host_format_check"] = {"build": build, "gpu_executed": False, + "cuda_code_compiled": False, + "scope": "Only binary input parsing and dimension/CFG validation; CUDA code excluded."} + require_success(build) + checks = [] + for path in packs: + checked = run([binary, path]) + checks.append(checked) + require_success(checked) + if json.loads(checked["stdout"])["gpu_executed"] is not False: + raise RuntimeError("host parser checker reported GPU execution") + original = packs[0].read_bytes() + for name, data in (("truncated", original[:-1]), ("wrong-magic", b"XXXX" + original[4:])): + path = directory / f"invalid-{name}.bin" + path.write_bytes(data) + checked = run([binary, path]) + checks.append(checked) + if checked["exit_code"] != 1: + raise RuntimeError(f"host format checker did not reject {name}") + report["host_format_check"].update(status="passed", checks=checks, binary_sha256=sha256(binary)) + + +def verify(report, args): + directory = ROOT / ".build/verification/cuda" + directory.mkdir(parents=True, exist_ok=True) + test_workloads = workloads() + packs = [] + report["workpacks"] = [] + for workload in test_workloads: + json_path = directory / f"{workload['name']}.json" + json_path.write_text(json.dumps(workload, indent=2) + "\n") + path = directory / f"{workload['name']}.bin" + path.write_bytes(pack(workload)) + packs.append(path) + report["workpacks"].append({"name": workload["name"], "functions": len(workload["functions"]), + "source_json_sha256": sha256(json_path), "binary_input_sha256": sha256(path), + "scope": "Generated analysis workload; not a full Rust crate or build."}) + host_format_check(report, directory, packs) + nvcc = report["tools"]["nvcc"]["path"] + if nvcc is None: + cmake = report["tools"]["cmake"]["path"] + if cmake: + report["cmake_configuration"] = run([cmake, "-S", ROOT / "cuda", "-B", directory / "cmake"]) + report.update(status="blocked", blockers=["nvcc is absent; CUDA source was not compiled.", + "No CUDA runtime device probe could run; no NVIDIA GPU execution is verified."]) + return 3 + report["nvcc_version"] = run([nvcc, "--version"]) + probe_source, probe_binary = directory / "device-probe.cu", directory / "device-probe" + probe_source.write_text(DEVICE_PROBE) + report["device_probe_build"] = run([nvcc, "-std=c++17", probe_source, "-o", probe_binary]) + require_success(report["device_probe_build"]) + report["device_probe_source_sha256"] = sha256(probe_source) + report["device_probe_binary_sha256"] = sha256(probe_binary) + report["device_probe_execution"] = run([probe_binary]) + probe = report["device_probe_execution"] + if probe["exit_code"] == 3: + report.update(status="blocked", blockers=[probe["stderr"].strip()]) + return 3 + require_success(probe) + report["device"] = json.loads(probe["stdout"]) + nvidia_smi = report["tools"]["nvidia-smi"]["path"] + if nvidia_smi: + report["nvidia_smi_identity"] = run([nvidia_smi, + "--query-gpu=index,name,uuid,pci.bus_id,driver_version", "--format=csv,noheader"]) + architecture = args.cuda_arch or f"sm_{report['device']['compute_major']}{report['device']['compute_minor']}" + binary = ROOT / ".build/cuda/cuda-liveness" + binary.parent.mkdir(parents=True, exist_ok=True) + report["cuda_build"] = run([nvcc, "-std=c++17", "-O2", f"-arch={architecture}", + ROOT / "cuda/liveness.cu", "-o", binary]) + require_success(report["cuda_build"]) + report["cuda_compiled"] = True + report["cuda_binary_sha256"] = sha256(binary) + for workload, path, record in zip(test_workloads, packs, report["workpacks"]): + output = path.with_suffix(".result.bin") + output.unlink(missing_ok=True) + record["cuda_execution"] = run([binary, path, output, args.repeats]) + require_success(record["cuda_execution"]) + actual, expected = read_result(output), oracle_words(workload) + metrics = json.loads(record["cuda_execution"]["stdout"]) + if (metrics["backend"] != "cuda" or metrics["device_name"] != report["device"]["name"] or + metrics["device_ordinal"] != 0 or metrics["converged"] is not True or + metrics["repeat_outputs_equal"] is not True or + metrics["gpu_execution_ms"]["median_ms"] <= 0): + raise RuntimeError("CUDA execution evidence is incomplete or inconsistent") + report["gpu_executed"] = True + if actual != expected: + raise RuntimeError(f"{workload['name']}: CUDA output differs from independent set oracle") + record.update(verified_against_independent_cpu_oracle=True, + verified_cells=len(actual), result_sha256=sha256(output), metrics=metrics) + report.update(status="verified_cuda_pass", cuda_compiled=True, gpu_executed=True, + verified_all_workpacks=True) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--report", type=Path, default=ROOT / "results/cuda-verification.json") + parser.add_argument("--repeats", type=int, default=7) + parser.add_argument("--cuda-arch", help="Override detected nvcc target, e.g. sm_89") + args = parser.parse_args() + if not 1 <= args.repeats <= 10000: + parser.error("--repeats must be between 1 and 10000") + report = {"recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "platform": {"system": platform.system(), "release": platform.release(), + "machine": platform.machine(), "python": platform.python_version()}, + "status": "not_run", "cuda_compiled": False, "gpu_executed": False, + "verified_all_workpacks": False, + "scope": "CUDA liveness analysis only. Does not verify a CUDA Rust compiler, its persistent adapter, full Cargo/NemoClaw builds, or speedups.", + "tools": {name: tool_identity(name) for name in ("nvcc", "nvidia-smi", "clang++", "g++", "cmake")}, + "sources_sha256": {str(path.relative_to(ROOT)): sha256(path) for path in + (ROOT / "cuda/liveness.cu", ROOT / "cuda/CMakeLists.txt", + ROOT / "scripts/export_cuda.py", Path(__file__).resolve())}, + "required_environment": "An authorized NVIDIA CUDA GPU host, compatible NVIDIA driver, nvcc toolkit, Python 3.9+, and a host C++ compiler. No remote host is selected or contacted by this script."} + try: + exit_code = verify(report, args) + except (RuntimeError, ValueError, KeyError, OSError) as error: + report.update(status="failed", error=str(error)) + exit_code = 1 + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"status": report["status"], "cuda_compiled": report["cuda_compiled"], + "gpu_executed": report["gpu_executed"], "report": str(args.report), + "blockers": report.get("blockers", []), "error": report.get("error")}, indent=2)) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/experiments/gpu-rust-compiler/scripts/verify_native_cuda.py b/experiments/gpu-rust-compiler/scripts/verify_native_cuda.py new file mode 100644 index 00000000000..3b7d78b6930 --- /dev/null +++ b/experiments/gpu-rust-compiler/scripts/verify_native_cuda.py @@ -0,0 +1,716 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify and benchmark persistent native CPU/CUDA compilation on one GPU host. + +Python controls requests and independently checks bitsets/program outputs. It +does not extract IR, solve production analysis, prune IR, or enter a measured +compilation path. Unavailable CUDA is blocked evidence, never a passing fallback. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import copy +from datetime import datetime, timezone +import json +import math +import os +from pathlib import Path +import random +import selectors +import shutil +import statistics +import subprocess +import time + +from benchmark_cuda import (GENERATOR_SEED, ORDER_SEED, hardware, related_evidence, + sha256, statistics_ms, synthetic_workloads) +from export_cuda import pack, read_result +from verify_cuda import (DEVICE_PROBE, oracle_words, require_success, run, + tool_identity, workloads as correctness_workloads) + +ROOT = Path(__file__).resolve().parents[1] + + +class NativeSession: + """One persistent Rust process; each response must identify that process.""" + def __init__(self, command, log_path): + self.command = [str(value) for value in command] + self.log = Path(log_path).open("w") + self.protocol_log = Path(log_path).with_suffix(".protocol.jsonl").open("w") + self.process = subprocess.Popen(self.command, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=self.log, + text=True, bufsize=1) + self.counter = 0 + self.pid = None + + def line(self, fields): + if any("\t" in str(field) or "\n" in str(field) for field in fields): + raise ValueError("Protocol field contains a delimiter") + started = time.perf_counter() + self.process.stdin.write("\t".join(str(field) for field in fields) + "\n") + self.process.stdin.flush() + selector = selectors.DefaultSelector() + try: + selector.register(self.process.stdout, selectors.EVENT_READ) + if not selector.select(180): + raise RuntimeError("Native request timed out") + line = self.process.stdout.readline() + finally: + selector.close() + elapsed = (time.perf_counter() - started) * 1000 + if not line: + raise RuntimeError(f"Native process exited without a response: {self.command}") + # Retain responses before parsing or oracle validation can fail. + self.protocol_log.write(json.dumps({"request": [str(field) for field in fields], + "response": line, "wall_ms": elapsed}) + "\n") + self.protocol_log.flush() + response = json.loads(line) + pid = response.get("process_id") + if pid is not None: + if self.pid is not None and pid != self.pid: + raise ValueError("Persistent process identity changed") + self.pid = pid + return response, elapsed + + def request(self, backend, algorithm, input_path, output_path, + residency="update", cpu_mode="auto", expected_error=False): + self.counter += 1 + identifier = str(self.counter) + response, elapsed = self.line((identifier, backend, algorithm, input_path, + output_path, residency, cpu_mode)) + if response.get("request_id") != identifier: + raise ValueError("Native response has wrong request identity") + if expected_error: + if response.get("status") != "error": + raise ValueError("Malformed request was accepted") + elif response.get("status") != "ok": + raise RuntimeError(response) + return response, elapsed + + def close(self): + try: + if self.process.poll() is None: + self.process.stdin.write("quit\n") + self.process.stdin.flush() + self.process.communicate(timeout=20) + if self.process.returncode: + raise RuntimeError(f"Native session exited {self.process.returncode}") + finally: + if self.process.poll() is None: + self.process.kill() + self.process.wait() + self.log.close() + self.protocol_log.close() + + +def validate_native(response, actual, expected, gpu_required=False, resident=False): + if actual != expected: + raise ValueError("Native output differs from independent set oracle") + for key in ("analysis_ms", "total_recurrent_ms"): + value = response.get(key) + if not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + raise ValueError(f"Missing positive native timing: {key}") + if not isinstance(response.get("cpu_pool_workers"), int): + raise ValueError("Reusable CPU pool identity is missing") + if gpu_required: + stats = response.get("gpu_stats") or {} + if (response.get("actual_gpu_functions", 0) < 1 or + response.get("cuda_context_creations") != 1 or + stats.get("gpu_ms", 0) <= 0 or + stats.get("resident_input") != int(resident) or + not response.get("capabilities", {}).get("device_name")): + raise ValueError("Actual persistent CUDA execution evidence is missing") + if resident and (stats.get("host_to_device_ms") != 0 or stats.get("staging_ms") != 0): + raise ValueError("Resident timing includes hidden input transfers") + + +def changed_facts(workload, shift=17, erase=False): + """Keep CFG shape while replacing facts, including facts that must disappear.""" + result = copy.deepcopy(workload) + result["name"] += f"-changed-{shift}" if not erase else "-cleared" + for function in result["functions"]: + count = function["value_count"] + for block in function["blocks"]: + for name in ("use", "def"): + block[name] = [] if erase or not count else [(value + shift) % count for value in block[name]] + for edge in function["phi_edge_uses"]: + edge["values"] = [] if erase or not count else [(value + shift) % count for value in edge["values"]] + return result + + +def prepare_pack(directory, workload): + path = directory / f"{workload['name']}.bin" + path.write_bytes(pack(workload)) + return path, oracle_words(workload) + + +def build_library(args, report, directory): + nvcc = shutil.which("nvcc") + if not nvcc: + report.update(status="blocked", blockers=["nvcc is unavailable; native CUDA library was not compiled."]) + return False + report["nvcc_version"] = run([nvcc, "--version"]) + probe_source = directory / "native-device-probe.cu" + probe = directory / "native-device-probe" + probe_source.write_text(DEVICE_PROBE) + report["device_probe_build"] = run([nvcc, "-std=c++17", probe_source, "-o", probe]) + require_success(report["device_probe_build"]) + report["device_probe_execution"] = run([probe]) + if report["device_probe_execution"]["exit_code"] == 3: + report.update(status="blocked", blockers=[report["device_probe_execution"]["stderr"].strip()]) + return False + require_success(report["device_probe_execution"]) + report["device"] = json.loads(report["device_probe_execution"]["stdout"]) + device = report["device"] + architecture = args.cuda_arch or f"sm_{device['compute_major']}{device['compute_minor']}" + args.cuda_library.parent.mkdir(parents=True, exist_ok=True) + report["native_library_build"] = run([nvcc, "-std=c++17", "-O3", f"-arch={architecture}", + "--shared", "-Xcompiler=-fPIC,-pthread", ROOT / "native/cuda_bridge.cu", "-o", args.cuda_library]) + require_success(report["native_library_build"]) + report["cuda_compiled"] = True + report["build_options"] = {"nvcc": report["native_library_build"]["command"], + "rust": "Hosted exact-head cargo build --locked --release; source identity and binary checksums retained by CI artifact."} + return True + + +def driver_command(args, profile=None): + command = [args.benchmark_binary, "--serve", "--threads", args.threads] + if not args.cpu_only: + command += ["--cuda-library", args.cuda_library] + if profile: + command += ["--route-profile", profile] + return command + + +def verify_lifecycle(args, report, directory): + session = NativeSession(driver_command(args), directory / "lifecycle.stderr.log") + records = [] + report["native_lifecycle_progress"] = [] + try: + # Grow, shrink and erase facts inside the same CUDA context/pinned buffers. + base = correctness_workloads()[0] + larger = synthetic_workloads()[2] + smaller = synthetic_workloads()[0] + tests = [base, changed_facts(base), larger, smaller, + changed_facts(smaller, erase=True), smaller] + for workload in tests: + path, expected = prepare_pack(directory, workload) + for alg in ("dense", "sparse") if not args.cpu_only else ("dense",): + output = directory / f"lifecycle-{len(records)}.result.bin" + response, wall = session.request("cpu" if args.cpu_only else "cuda", alg, path, output) + report["native_lifecycle_progress"].append({"name": workload["name"], + "algorithm": alg, "report": response, "external_request_ms": wall}) + if response.get("actual_gpu_functions", 0) > 0 and response.get("gpu_stats", {}).get("gpu_ms", 0) > 0: + report["gpu_executed"] = True + validate_native(response, read_result(output), expected, not args.cpu_only) + if not args.cpu_only: + cap = response["capabilities"] + if (cap.get("device_name") != report["device"]["name"] or + cap.get("device_uuid") != report["device"]["uuid_hex"] or + cap.get("driver_version") != report["device"]["driver_version"] or + cap.get("runtime_version") != report["device"]["runtime_version"]): + raise ValueError("Native CUDA device identity differs from independent runtime probe") + records.append({"name": workload["name"], "algorithm": alg, + "input_sha256": sha256(path), "output_sha256": sha256(output), + "verified_cells": len(expected), "external_request_ms": wall, "report": response}) + if not args.cpu_only: + replay, replay_wall = session.request("cuda", alg, path, output, "resident") + validate_native(replay, read_result(output), expected, True, True) + if replay["gpu_stats"].get("reused_context") != 1: + raise ValueError("CUDA resident replay failed to reuse context") + records.append({"name": workload["name"], "algorithm": alg, "residency": "resident", + "external_request_ms": replay_wall, "report": replay}) + if not args.cpu_only and workload is smaller: + for alg in ("dense", "sparse"): + checked, wall = session.request("abi-check", alg, path, output) + validate_native(checked, read_result(output), expected, True) + if (checked.get("second_pending_submit_rejected") is not True or + checked.get("input_snapshot_ownership_checked") is not True or + checked["gpu_stats"].get("device_allocations") != 0 or + checked["gpu_stats"].get("pinned_allocations") != 0): + raise ValueError("Pending/ownership/high-water buffer reuse contract failed") + records.append({"native_adapter_contract": True, "external_request_ms": wall, + "report": checked}) + invalid = directory / "malformed.bin" + invalid.write_bytes(b"GLC1\0") + rejected = directory / "must-not-exist.result.bin" + rejected.unlink(missing_ok=True) + failure, _ = session.request("cuda" if not args.cpu_only else "cpu", "dense", invalid, + rejected, expected_error=True) + if rejected.exists(): + raise ValueError("Malformed request created a result file") + path, expected = prepare_pack(directory, smaller) + output = directory / "recovered.result.bin" + recovered, _ = session.request("cpu" if args.cpu_only else "cuda", "sparse", path, output) + validate_native(recovered, read_result(output), expected, not args.cpu_only) + records += [{"malformed_request_rejected": failure}, {"recovery_report": recovered}] + if not args.cpu_only: + changed, _ = prepare_pack(directory, changed_facts(smaller)) + failure, _ = session.request("cuda", "dense", changed, rejected, + "resident", expected_error=True) + records.append({"changed_resident_request_rejected": failure}) + for mixed in (correctness_workloads()[1], synthetic_workloads()[-1]): + path, expected = prepare_pack(directory, mixed) + for alg in ("dense", "sparse"): + output = directory / f"forced-{mixed['name']}-{alg}.result.bin" + response, wall = session.request("forced-hybrid", alg, path, output) + validate_native(response, read_result(output), expected, True) + if (response.get("actual_cpu_functions", 0) == 0 or + response.get("gpu_submitted_before_cpu") is not True or + response.get("cpu_work_before_gpu_finish") is not True): + raise ValueError("Mixed CPU/GPU submission ordering was not proved") + records.append({"forced_mixed": True, "algorithm": alg, + "external_request_ms": wall, "report": response}) + if not any(record.get("forced_mixed") and + record["report"].get("gpu_event_pending_before_cpu") is True + for record in records): + raise ValueError("No forced mixed case proved GPU request unfinished when CPU work began") + report["lifecycle"] = {"verified": True, "same_process_id": session.pid, "records": records, + "checks": ["changed facts replace obsolete facts", "buffer growth/shrink", "dense/sparse equality", + "malformed request rejection and subsequent recovery", "CPU/GPU independent subset scatter"], + "overlap_limit": "CUDA completion event is queried before and after CPU work; at least one mixed case remains unfinished when CPU work begins. No GPU timeline is collected to prove kernel/CPU overlap duration."} + finally: + session.close() + + +def summarize_requests(records): + return {"external_request_ms": statistics_ms([record["external_request_ms"] for record in records]), + "native_recurrent_ms": statistics_ms([record["report"]["total_recurrent_ms"] for record in records]), + "native_analysis_ms": statistics_ms([record["report"]["analysis_ms"] for record in records]), + "reports": records} + + +def benchmark_fresh_analysis(args, directory, input_path, expected, calibration): + """Actual cold processes, including library/context and CPU-pool creation.""" + modes = [("cpu", "dense")] + if not args.cpu_only: + modes += [("cuda", "dense"), ("cuda", "sparse")] + result = {} + selected = calibration["cpu_calibration"] + for backend, algorithm in modes: + samples = [] + for repeat in range(args.fresh_repeats): + output = directory / f"cold-{input_path.stem}-{backend}-{algorithm}-{repeat}.result.bin" + started = time.perf_counter() + session = NativeSession(driver_command(args), directory / "cold.stderr.log") + try: + response, request_wall = session.request(backend, algorithm, input_path, output, + "update", f"{selected['selected_mode']}@{selected['selected_workers']}") + wall = (time.perf_counter() - started) * 1000 + validate_native(response, read_result(output), expected, backend == "cuda") + samples.append({"external_process_to_response_ms": wall, + "external_request_ms": request_wall, "report": response}) + finally: + session.close() + result[f"{backend}/{algorithm}"] = { + "cold_external_ms": statistics_ms([sample["external_process_to_response_ms"] for sample in samples]), + "samples": samples, "scope": "Fresh Rust process, reusable CPU pool creation, input read/decode, library/context creation when CUDA, analysis and response serialization. Output checks and process shutdown outside timer."} + return result + + +def benchmark_analysis(args, report, directory): + session = NativeSession(driver_command(args), directory / "benchmark.stderr.log") + profile = directory / "measured-route-profile.csv" + profile.unlink(missing_ok=True) + records = [] + rng = random.Random(ORDER_SEED) + try: + for workload in synthetic_workloads(): + if args.workloads and workload["name"] not in args.workloads: + continue + # Held-out requests rotate value IDs by a different fixed amount. + # The graph stays identical; this does not test unseen graph shapes. + variants = [workload, changed_facts(workload, 17), changed_facts(workload, 43)] + packs = [prepare_pack(directory, variant) for variant in variants] + path, expected = packs[0] + output = directory / "analysis.result.bin" + calibration, _ = session.request("calibrate", "dense", path, output) + validate_native(calibration, read_result(output), expected) + selected_cpu = calibration["cpu_calibration"]["selected_mode"] + modes = ["cpu-serial", "cpu-function_pool", "cpu-word_pool"] + if not args.cpu_only: + modes += ["cuda-dense-update", "cuda-sparse-update", + "cuda-dense-resident", "cuda-sparse-resident"] + if workload["name"] == "mixed-compact-and-long": + modes += ["mixed-dense-update", "mixed-sparse-update"] + measured = {mode: [] for mode in modes} + first = {} + count_train = args.repeats // 2 + invocation_order = [] + for repeat in range(args.repeats + 1): + fact_index = repeat % 2 if repeat <= count_train else 2 + path, expected = packs[fact_index] + order = modes.copy() + rng.shuffle(order) + invocation_order.append(order) + for mode in order: + if mode.startswith("cpu-"): + cpu_name = mode.removeprefix("cpu-") + response, wall = session.request("cpu", "dense", path, output, "update", cpu_name) + validate_native(response, read_result(output), expected) + else: + _, algorithm, residency = mode.split("-") + if residency == "resident": + # Seed immediately before replay; upload timing is + # retained separately by the update variant. + seeded, _ = session.request("cuda", algorithm, path, output) + validate_native(seeded, read_result(output), expected, True) + response, wall = session.request("forced-hybrid" if mode.startswith("mixed-") else "cuda", + algorithm, path, output, residency, selected_cpu) + validate_native(response, read_result(output), expected, True, residency == "resident") + detail = {"external_request_ms": wall, "report": response, + "facts_variant": fact_index, + "sample_set": "training" if repeat <= count_train else "heldout"} + if repeat == 0: + first[mode] = detail + else: + measured[mode].append(detail) + record = {"name": workload["name"], "scope": "Synthetic native liveness pass, not a Rust crate.", + "functions": len(workload["functions"]), "input_sha256": [sha256(path) for path, _ in packs], + "cpu_selected_variant": selected_cpu, "cpu_calibration": calibration, + "first_requests": first, "invocation_order": invocation_order, + "results": {mode: summarize_requests(values) for mode, values in measured.items()}} + record["fresh_processes"] = benchmark_fresh_analysis(args, directory, packs[0][0], packs[0][1], calibration) + if not args.cpu_only: + cpu_samples = measured[f"cpu-{selected_cpu}"] + cpu_ms = statistics.median([sample["report"]["total_recurrent_ms"] for sample in cpu_samples]) + comparisons = {} + for algorithm in ("dense", "sparse"): + update = measured[f"cuda-{algorithm}-update"] + resident = measured[f"cuda-{algorithm}-resident"] + update_ms = statistics.median([sample["report"]["total_recurrent_ms"] for sample in update]) + resident_ms = statistics.median([sample["report"]["analysis_ms"] for sample in resident]) + comparisons[algorithm] = {"cpu_native_recurrent_median_ms": cpu_ms, + "cuda_native_recurrent_median_ms": update_ms, + "cpu_over_cuda_recurrent_ratio": cpu_ms / update_ms, + "cuda_resident_analysis_median_ms": resident_ms, + "ratio_scope": "Update-inclusive request: input file read, native GLC decode/reverse-edge construction, native solve/pinned staging/transfers/synchronization/result assembly, and output file write. JSON/IPC overhead measured separately."} + record["comparison"] = comparisons + # Select algorithm using training only. The shared Rust profile + # rejects noisy wins and requires a separate held-out margin. + chosen = min(("dense", "sparse"), key=lambda alg: statistics.median( + [sample["report"]["total_recurrent_ms"] for sample in measured[f"cuda-{alg}-update"] + if sample["sample_set"] == "training"])) + gpu_samples = measured[f"cuda-{chosen}-update"] + arrays = [] + for samples, sample_set in ((cpu_samples, "training"), (gpu_samples, "training"), + (cpu_samples, "heldout"), (gpu_samples, "heldout")): + arrays.append(",".join(str(sample["report"]["total_recurrent_ms"]) + for sample in samples if sample["sample_set"] == sample_set)) + response, _ = session.line(("profile", packs[0][0], selected_cpu, chosen, *arrays, profile)) + if response.get("status") != "ok": + raise RuntimeError(response) + record["calibration_profile_record"] = response + if workload["name"] == "mixed-compact-and-long": + chosen_mixed = min(("dense", "sparse"), key=lambda alg: statistics.median( + [sample["report"]["total_recurrent_ms"] for sample in measured[f"mixed-{alg}-update"] + if sample["sample_set"] == "training"])) + mixed_samples = measured[f"mixed-{chosen_mixed}-update"] + mixed_arrays = [] + for samples, sample_set in ((cpu_samples, "training"), (mixed_samples, "training"), + (cpu_samples, "heldout"), (mixed_samples, "heldout")): + mixed_arrays.append(",".join(str(sample["report"]["total_recurrent_ms"]) + for sample in samples if sample["sample_set"] == sample_set)) + indices = ",".join(str(index) for index, function in enumerate(workload["functions"]) + if len(function["blocks"]) <= 64) + response, _ = session.line(("profile", packs[0][0], selected_cpu, chosen_mixed, + *mixed_arrays, profile, indices)) + if response.get("status") != "ok": + raise RuntimeError(response) + record["mixed_context_calibration"] = response + records.append(record) + report["analysis_workloads"] = records + report["analysis_timing_scope"] = { + "update": "Repeated native request including read, decode, predecessor packing, pool scheduling OR pinned staging/H2D/kernel/wait/D2H, assembly and result-file write; no Python analysis.", + "resident": "Identical prior GPU input replay; no packing/staging/H2D. Result is reset from prior facts, returned and checked. Presented separately, never used as a routing-speedup claim.", + "cpu": "Same-process native serial/function-parallel/word-parallel worklists with a reusable worker pool. Mode selected by independent calibration samples; worker startup recorded separately.", + "verification": "Every result cell compared against independent set oracle after timing; input mutations include removal of old bits.", + "routing": "Training and held-out arrays are separate; conservative profile requires GPU p90 below CPU p10 by 10% in both. Missing/noisy/unknown shapes fall back to CPU."} + report["analysis_timing_scope"]["heldout_limit"] = ( + "Held-out timings use deterministic value-ID rotations on identical CFGs, " + "not independent graph geometries. No broader generalization claim is made.") + if not args.cpu_only: + report["route_profile"] = {"path": str(profile), "sha256": sha256(profile), + "text": profile.read_text()} + finally: + session.close() + if not args.cpu_only: + adaptive = NativeSession(driver_command(args, profile), directory / "adaptive.stderr.log") + adaptive_records = [] + try: + for workload in synthetic_workloads(): + if args.workloads and workload["name"] not in args.workloads: + continue + changed = changed_facts(workload, 71) + path, expected = prepare_pack(directory, changed) + output = directory / "adaptive.result.bin" + for _ in range(5): + response, wall = adaptive.request("hybrid", "dense", path, output) + validate_native(response, read_result(output), expected, + response.get("actual_gpu_functions", 0) > 0) + adaptive_records.append({"name": workload["name"], + "external_request_ms": wall, "report": response}) + report["adaptive_routing"] = {"records": adaptive_records, + "cpu_only_decisions": sum(record["report"]["actual_gpu_functions"] == 0 for record in adaptive_records), + "gpu_decisions": sum(record["report"]["actual_gpu_functions"] > 0 for record in adaptive_records), + "scope": "Additional held-out value facts. CPU-only is a valid measured result, not GPU acceleration."} + finally: + adaptive.close() + return profile if not args.cpu_only else None + + +def verify_program_output(path, expected): + execution = run([path], timeout=30) + require_success(execution) + if execution["stdout"] != expected or execution["stderr"]: + raise ValueError(f"Compiled program differs from standard rustc reference: {execution}") + + +def compiler_command(args, mode, profile=None, serve=False, cpu_workers=None, objective="latency"): + backend, algorithm = mode.split("/", 1) + command = [args.compiler_binary, "--backend", backend] + command += ["--cpu-workers", args.threads if cpu_workers is None else cpu_workers, + "--mode", objective] + if backend.startswith("cuda"): + command += ["--cuda-algorithm", algorithm, "--cuda-library", args.cuda_library] + if profile and backend == "cuda-hybrid": + command += ["--route-profile", profile] + if serve: + command += ["--serve"] + return command + + +def benchmark_compilation(args, report, directory, profile): + if args.skip_compilation: + report["complete_compilation"] = {"status": "skipped_explicitly"} + return + if not args.references.is_dir(): + raise RuntimeError(f"Standard rustc references unavailable: {args.references}") + programs = (args.references / "programs.txt").read_text().splitlines() + reference_files = [path for path in args.references.iterdir() if path.is_file()] + report["standard_rust_references"] = { + "rustc_identity": (args.references / "rustc-version.txt").read_text(), + "sha256": {path.name: sha256(path) for path in sorted(reference_files)}, + "source": "Compiled and executed by pinned standard rustc on hosted CPU job; immutable artifact verified before GPU job."} + modes = ["cpu/dense"] + if not args.cpu_only: + modes += ["cuda/dense", "cuda/sparse", "cuda-hybrid/dense"] + workers = {mode: NativeSession(compiler_command(args, mode, profile, True), + directory / f"compile-{mode.replace('/', '-')}.stderr.log") for mode in modes} + rng = random.Random(ORDER_SEED + 2) + results = [] + cold = [] + try: + for name in programs: + source = args.references / f"{name}.rs" + expected = (args.references / f"{name}.expected.txt").read_text() + output = directory / f"program-{name}" + detail_path = directory / f"program-{name}.json" + measured = {f"{mode}/{scope}": [] for mode in modes for scope in ("fresh", "persistent")} + for mode in modes: + response, wall = workers[mode].line(("compile", source, output, detail_path)) + if response.get("status") != "ok": + raise RuntimeError(response) + verify_program_output(output, expected) + cold.append({"program": name, "mode": mode, "external_request_ms": wall, "report": response}) + order = list(measured) + for repeat in range(args.compile_repeats): + rng.shuffle(order) + for key in order: + backend, algorithm, scope = key.split("/") + mode = f"{backend}/{algorithm}" + if scope == "persistent": + response, wall = workers[mode].line(("compile", source, output, detail_path)) + if response.get("status") != "ok": + raise RuntimeError(response) + else: + command = compiler_command(args, mode, profile) + [source, "--output", output, + "--report", detail_path] + start = time.perf_counter() + compiled = run(command, timeout=180) + wall = (time.perf_counter() - start) * 1000 + require_success(compiled) + response = json.loads(detail_path.read_text()) + verify_program_output(output, expected) + if backend == "cuda": + if response.get("actual_gpu_functions", 0) < 1: + raise ValueError("Explicit native CUDA compilation did not execute GPU") + if scope == "persistent": + stats = response.get("cuda_stats") or {} + if (response.get("gpu_pipeline_creations") != 1 or + stats.get("reused_context") != 1 or + stats.get("device_allocations") != 0 or + stats.get("pinned_allocations") != 0): + raise ValueError("Complete compiler did not reuse native CUDA context/buffers") + # Root compiler emits lifecycle statistics using the same + # in-process context. Keep every report for exact proof. + measured[key].append({"external_compile_ms": wall, "report": response, + "output_sha256": sha256(output), "expected_output": expected}) + results.append({"name": name, "source_sha256": sha256(source), "expected_output": expected, + "results": {key: {"external_compile_ms": statistics_ms([sample["external_compile_ms"] for sample in samples]), + "reports": samples} for key, samples in measured.items()}}) + report["complete_compilation"] = {"status": "verified", "programs": results, "initial_requests": cold, + "scope": "Complete compilation of supported scalar Rust programs, including frontend, native analysis, pruning, LLVM emission, Clang codegen/link, reporting and IPC/process startup. Executable verification occurs outside timer. No Python analysis path.", + "limitations": "Not a full Rust compiler or NemoClaw/Omarchy Cargo build. Standard rustc executes references on hosted CPU hardware, so its build time is not compared with GPU-runner compilation times."} + finally: + for worker in workers.values(): + worker.close() + benchmark_compile_throughput(args, report, directory, programs, modes) + + +def benchmark_compile_throughput(args, report, directory, programs, modes): + """Concurrent, persistent compiler processes with equal CPU thread budgets.""" + concurrency = args.concurrency + available = report["analysis_workloads"][0]["cpu_calibration"]["cpu_pool_workers"] + pool_budget = max(1, available // concurrency) + cohort = programs * concurrency + worker_sets = {} + samples = {mode: [] for mode in modes} + warmup = [] + seed = ORDER_SEED + 3 + rng = random.Random(seed) + cohort_orders = [] + try: + for mode in modes: + workers = [NativeSession(compiler_command(args, mode, None, True, pool_budget, "throughput"), + directory / f"throughput-{mode.replace('/', '-')}-{index}.stderr.log") + for index in range(concurrency)] + worker_sets[mode] = workers + # Warm every process/context at the largest supported fixture shape. + for index, worker in enumerate(workers): + for name in programs: + source = args.references / f"{name}.rs" + output = directory / f"throughput-warm-{mode.replace('/', '-')}-{index}-{name}" + detail = output.with_suffix(".json") + response, wall = worker.line(("compile", source, output, detail)) + if response.get("status") != "ok": + raise RuntimeError(response) + verify_program_output(output, (args.references / f"{name}.expected.txt").read_text()) + warmup.append({"mode": mode, "worker": index, "program": name, + "external_request_ms": wall, "report": response}) + for repeat in range(args.throughput_repeats): + order = list(modes) + rng.shuffle(order) + cohort_orders.append(order) + for mode in order: + workers = worker_sets[mode] + def compile_partition(index): + completed = [] + for job in range(index, len(cohort), concurrency): + name = cohort[job] + source = args.references / f"{name}.rs" + output = directory / f"throughput-{mode.replace('/', '-')}-{repeat}-{job}" + detail = output.with_suffix(".json") + response, elapsed = workers[index].line(("compile", source, output, detail)) + if response.get("status") != "ok": + raise RuntimeError(response) + if response.get("cpu_worker_threads") != pool_budget: + raise ValueError("Concurrent compiler CPU budget differs between modes") + if mode.startswith("cuda/") and response.get("actual_gpu_functions", 0) < 1: + raise ValueError("Concurrent explicit CUDA compiler did not execute GPU") + if mode.startswith("cuda-hybrid/") and response.get("actual_gpu_functions") != 0: + raise ValueError("Throughput routing used uncalibrated GPU policy") + completed.append({"job": job, "program": name, "external_request_ms": elapsed, + "report": response, "output": str(output)}) + return completed + with ThreadPoolExecutor(max_workers=concurrency) as executor: + start = time.perf_counter() + futures = [executor.submit(compile_partition, index) for index in range(concurrency)] + completed = [job for future in futures for job in future.result()] + wall = (time.perf_counter() - start) * 1000 + for job in completed: + expected = (args.references / f"{job['program']}.expected.txt").read_text() + verify_program_output(Path(job["output"]), expected) + samples[mode].append({"cohort_wall_ms": wall, "jobs_per_second": len(cohort) * 1000 / wall, + "request_ms": statistics_ms([job["external_request_ms"] for job in completed]), + "jobs": completed, "all_outputs_match_standard_rustc": True}) + finally: + for workers in worker_sets.values(): + for worker in workers: + worker.close() + records = [] + for mode, measured in samples.items(): + wall_stats = statistics_ms([sample["cohort_wall_ms"] for sample in measured]) + records.append({"mode": mode, "process_count": concurrency, "cpu_workers_per_process": pool_budget, + "jobs_per_cohort": len(cohort), "cohort_wall_ms": wall_stats, + "jobs_per_second_at_median_wall": len(cohort) * 1000 / wall_stats["median_ms"], + "samples": measured}) + report["complete_compile_throughput"] = {"results": records, + "order_seed": seed, "cohort_invocation_order": cohort_orders, + "warmup_requests": warmup, + "scope": "Same cohort, same machine, concurrent persistent native compiler processes, equal CPU worker budgets. Includes compilation/reporting; output execution checked after cohort timer.", + "routing": "No concurrent throughput calibration exists. No latency profile is supplied to throughput mode; cuda-hybrid must visibly use CPU fallback.", + "limits": "Supported scalar programs only; process/GPU contention measured here does not establish full NemoClaw build throughput."} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler-binary", type=Path, default=ROOT / ".build/compiler/release/gpu-rust-compiler") + parser.add_argument("--benchmark-binary", type=Path, default=ROOT / ".build/compiler/release/gpu-native-benchmark") + parser.add_argument("--cuda-library", type=Path, default=ROOT / ".build/cuda/libgpulab_cuda.so") + parser.add_argument("--references", type=Path, default=ROOT / ".build/ci-native/references") + parser.add_argument("--repeats", type=int, default=21) + parser.add_argument("--compile-repeats", type=int, default=9) + parser.add_argument("--fresh-repeats", type=int, default=3) + parser.add_argument("--throughput-repeats", type=int, default=5) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--threads", type=int, default=0, + help="Reusable native CPU pool size; 0 uses all available CPUs") + parser.add_argument("--report", type=Path, default=ROOT / "results/native-cuda-verification.json") + parser.add_argument("--cuda-arch") + parser.add_argument("--cpu-only", action="store_true") + parser.add_argument("--skip-compilation", action="store_true") + parser.add_argument("--workloads", nargs="+", choices=[case["name"] for case in synthetic_workloads()]) + args = parser.parse_args() + if not 11 <= args.repeats <= 1000: + parser.error("--repeats must be 11..=1000 to retain independent training and held-out samples") + if not 1 <= args.compile_repeats <= 100 or not 0 <= args.threads <= 256: + parser.error("--compile-repeats must be 1..=100; --threads must be 0..=256") + if not 1 <= args.throughput_repeats <= 100 or not 1 <= args.concurrency <= 16: + parser.error("--throughput-repeats must be 1..=100; --concurrency must be 1..=16") + if not 1 <= args.fresh_repeats <= 100: + parser.error("--fresh-repeats must be 1..=100") + directory = ROOT / ".build/verification/native-cuda" + directory.mkdir(parents=True, exist_ok=True) + report = {"status": "not_run", "recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "source_revision": os.environ.get("CANDIDATE_SHA") or os.environ.get("GITHUB_SHA"), + "hardware": hardware(), "repeats": args.repeats, "compile_repeats": args.compile_repeats, + "gpu_executed": False, "cuda_compiled": False, "generator_seed": GENERATOR_SEED, + "order_seed": ORDER_SEED, "threads": args.threads, "related_evidence": related_evidence(), + "scope": "Non-shipping persistent native CUDA compiler experiment; supported scalar programs and synthetic analyses only.", + "tools": {name: tool_identity(name) for name in ("nvcc", "clang", "nvidia-smi")}} + exit_code = 1 + try: + if not args.cpu_only and not build_library(args, report, directory): + exit_code = 3 + else: + binaries = {"compiler": args.compiler_binary, "benchmark": args.benchmark_binary} + if not args.cpu_only: + binaries["native_cuda_library"] = args.cuda_library + report["binaries"] = {name: {"path": str(path.resolve()), "sha256": sha256(path)} + for name, path in binaries.items()} + sources = list((ROOT / "compiler/src").rglob("*.rs")) + [ROOT / "native/cuda_bridge.cu", + ROOT / "native/cuda_bridge.h", Path(__file__).resolve()] + report["implementation_sha256"] = {str(path.relative_to(ROOT)): sha256(path) + for path in sources if path.is_file()} + verify_lifecycle(args, report, directory) + report.update(gpu_executed=not args.cpu_only, native_lifecycle_verified=True) + profile = benchmark_analysis(args, report, directory) + benchmark_compilation(args, report, directory, profile) + report.update(status="cpu_only_verified" if args.cpu_only else "verified_native_cuda", + gpu_executed=not args.cpu_only, all_outputs_independently_verified=True) + exit_code = 0 + except (OSError, RuntimeError, ValueError, KeyError, subprocess.TimeoutExpired) as error: + report.update(status="failed", error=str(error)) + finally: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({"status": report["status"], "report": str(args.report), + "gpu_executed": report["gpu_executed"], "error": report.get("error"), + "blockers": report.get("blockers", [])}, indent=2)) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/gpu-rust-compiler/src/liveness.metal b/experiments/gpu-rust-compiler/src/liveness.metal new file mode 100644 index 00000000000..40482eec93f --- /dev/null +++ b/experiments/gpu-rust-compiler/src/liveness.metal @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +using namespace metal; + +// Each group owns a function and one 32-value word. Words and functions are +// independent, so all fixed-point barriers stay inside this threadgroup. +struct Group { + uint block_base; + uint block_count; + uint row_base; + uint words; + uint word; +}; + +kernel void liveness( + device const Group *groups [[buffer(0)]], + device const uint *successor_offsets [[buffer(1)]], + device const uint *successors [[buffer(2)]], + device const uint *uses [[buffer(3)]], + device const uint *defs [[buffer(4)]], + device const uint *phi_out [[buffer(5)]], + device uint *result [[buffer(6)]], + device uint *iterations [[buffer(7)]], + device uint *converged [[buffer(8)]], + threadgroup uint *previous [[threadgroup(0)]], + threadgroup uint *next [[threadgroup(1)]], + uint3 group_id [[threadgroup_position_in_grid]], + uint lane [[thread_index_in_threadgroup]], + uint3 group_size [[threads_per_threadgroup]]) { + Group group = groups[group_id.x]; + threadgroup atomic_uint changed; + for (uint block = lane; block < group.block_count; block += group_size.x) { + previous[block] = 0; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + bool stable = false; + uint sweeps = 0; + // With zero initialization each bit propagates along a simple CFG path. + // N + 1 sweeps includes the final no-change sweep, even with cycles. + for (uint round = 0; round <= group.block_count; ++round) { + if (lane == 0) atomic_store_explicit(&changed, 0, memory_order_relaxed); + threadgroup_barrier(mem_flags::mem_threadgroup); + for (uint block = lane; block < group.block_count; block += group_size.x) { + uint global_block = group.block_base + block; + uint cell = group.row_base + block * group.words + group.word; + uint live_out = phi_out[cell]; + for (uint edge = successor_offsets[global_block]; + edge < successor_offsets[global_block + 1]; ++edge) { + live_out |= previous[successors[edge] - group.block_base]; + } + uint value = uses[cell] | (live_out & ~defs[cell]); + next[block] = value; + if (value != previous[block]) { + atomic_fetch_or_explicit(&changed, 1, memory_order_relaxed); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + sweeps = round + 1; + if (atomic_load_explicit(&changed, memory_order_relaxed) == 0) { + stable = true; + break; + } + for (uint block = lane; block < group.block_count; block += group_size.x) { + previous[block] = next[block]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + for (uint block = lane; block < group.block_count; block += group_size.x) { + uint cell = group.row_base + block * group.words + group.word; + result[cell] = previous[block]; + } + if (lane == 0) { + iterations[group_id.x] = sweeps; + converged[group_id.x] = stable ? 1 : 0; + } +} diff --git a/experiments/gpu-rust-compiler/tests/test_benchmark_cuda.py b/experiments/gpu-rust-compiler/tests/test_benchmark_cuda.py new file mode 100644 index 00000000000..a43bd7e3c7b --- /dev/null +++ b/experiments/gpu-rust-compiler/tests/test_benchmark_cuda.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmark evidence must keep CPU, GPU and process timing boundaries distinct.""" +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from benchmark_cuda import comparisons, statistics_ms, synthetic_workloads, validate_pass + + +def metrics(backend, samples): + return {"backend": backend, "repeat_outputs_equal": True, "repeats": len(samples), + "warm_pass_end_to_end_ms": statistics_ms(samples)} + + +class CudaBenchmarkTests(unittest.TestCase): + def test_workloads_cover_large_batches_and_long_dependency_chains_reproducibly(self): + first = synthetic_workloads() + self.assertEqual(first, synthetic_workloads()) + self.assertEqual([workload["name"] for workload in first], + ["tiny", "compact-batch-64", "compact-batch-512", "compact-batch", + "few-wide", "long-chain", "mixed-compact-and-long"]) + large = first[3] + cells = sum(len(function["blocks"]) * ((function["value_count"] + 31) // 32) + for function in large["functions"]) + self.assertGreaterEqual(cells, 500_000) + self.assertGreaterEqual(len(large["functions"]), 4000) + chain = first[5]["functions"][0] + self.assertEqual(chain["blocks"][0]["successors"], [1]) + self.assertEqual(chain["blocks"][-1]["successors"], []) + self.assertEqual(chain["blocks"][-1]["use"], [0, 32, 64]) + + def test_pass_rejects_wrong_output_or_incomplete_gpu_evidence(self): + cpu = metrics("cpu-native", [1.0, 2.0, 3.0]) + validate_pass(cpu, "cpu-native", [1, 2], [1, 2], 3) + with self.assertRaisesRegex(ValueError, "oracle"): + validate_pass(cpu, "cpu-native", [1, 3], [1, 2], 3) + cuda = metrics("cuda", [1.0, 2.0, 3.0]) + with self.assertRaisesRegex(ValueError, "GPU"): + validate_pass(cuda, "cuda", [1, 2], [1, 2], 3) + cuda.update(converged=True, device_name="test-device", + gpu_execution_ms=statistics_ms([0.2, 0.3, 0.4])) + validate_pass(cuda, "cuda", [1, 2], [1, 2], 3) + + def test_comparison_keeps_input_update_and_resident_input_separate(self): + cpu = metrics("cpu-native", [5.0, 6.0, 7.0]) + cpu["selected_variant"] = "serial" + cuda = metrics("cuda", [1.0, 2.0, 3.0]) + cuda["warm_input_update_end_to_end_ms"] = statistics_ms([7.0, 8.0, 9.0]) + result = comparisons(cpu, cuda) + self.assertEqual(result["cpu_selected_variant"], "serial") + self.assertEqual(result["cpu_over_cuda_resident_input_ratio"], 3.0) + self.assertEqual(result["cpu_over_cuda_input_update_ratio"], 0.75) + self.assertIn("excludes host-to-device", result["resident_input_scope"]) + + def test_cpu_only_records_missing_gpu_without_speedup_or_zero_time(self): + result = comparisons(metrics("cpu-native", [1.0]), None) + self.assertEqual(result["cuda_status"], "unavailable") + self.assertIsNone(result["cpu_over_cuda_resident_input_ratio"]) + self.assertIsNone(result["cuda_resident_input_median_ms"]) + with self.assertRaisesRegex(ValueError, "positive"): + statistics_ms([0.0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/gpu-rust-compiler/tests/test_liveness_reference.py b/experiments/gpu-rust-compiler/tests/test_liveness_reference.py new file mode 100644 index 00000000000..8cf53ea19fe --- /dev/null +++ b/experiments/gpu-rust-compiler/tests/test_liveness_reference.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Portable CPU oracle correctness and native/CUDA workload-layout compatibility.""" +import copy +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from liveness_reference import solve_reference +from verify_cuda import oracle_words, workloads + + +def block(successors=None, use=None, definitions=None): + return {"name": "block", "successors": successors or [], + "use": use or [], "def": definitions or []} + + +def fixture(): + return {"version": 1, "functions": [{"name": "fixture", "value_count": 33, + "blocks": [block([1], [0], [32]), block([0], [32], [0])], + "phi_edge_uses": [{"from": 0, "to": 1, "values": [32]}]}]} + + +class LivenessReferenceTests(unittest.TestCase): + def test_cyclic_cfg_and_phi_uses_are_killed_by_predecessor_definitions(self): + result = solve_reference(fixture()) + self.assertEqual(result["live_in_words"], [1, 0, 0, 1]) + self.assertEqual(result["row_offsets"], [0, 2, 4]) + self.assertEqual(result["function_block_offsets"], [0, 2]) + self.assertEqual(result["requested_backend"], "cpu-reference") + self.assertEqual(result["actual_gpu_functions"], 0) + + def test_multi_function_layout_zero_values_and_word_boundaries(self): + workload = {"version": 1, "functions": [ + {"name": "zero", "value_count": 0, "blocks": [block()], "phi_edge_uses": []}, + {"name": "wide", "value_count": 33, + "blocks": [block([1], [31]), block(use=[32])], "phi_edge_uses": []}]} + result = solve_reference(workload) + self.assertEqual(result["live_in_words"], [0, 1 << 31, 1, 0, 1]) + self.assertEqual(result["row_offsets"], [0, 1, 3, 5]) + self.assertEqual(result["function_block_offsets"], [0, 1, 3]) + + def test_rejects_invalid_dimensions_ids_successors_and_phi_edges(self): + cases = [] + for key, value in (("version", 2), ("version", True), ("functions", [])): + candidate = fixture() + candidate[key] = value + cases.append(candidate) + for value in (-1, True, 1_000_000): + candidate = fixture() + candidate["functions"][0]["value_count"] = value + cases.append(candidate) + candidate = fixture() + candidate["functions"][0]["blocks"] = [] + cases.append(candidate) + for field, values in (("use", [33]), ("def", [-1]), ("use", [True]), + ("successors", [2]), ("successors", [-1]), ("successors", [False])): + candidate = fixture() + candidate["functions"][0]["blocks"][0][field] = values + cases.append(candidate) + for field, value in (("from", -1), ("to", 0), ("to", 2), ("from", True), ("values", [33])): + candidate = fixture() + candidate["functions"][0]["phi_edge_uses"][0][field] = value + cases.append(candidate) + candidate = fixture() + del candidate["functions"][0]["blocks"][0]["def"] + cases.append(candidate) + for index, candidate in enumerate(cases): + with self.subTest(index=index), self.assertRaises(ValueError): + solve_reference(candidate) + + def test_rejects_total_cell_limit_before_attempting_large_solution(self): + function = {"value_count": 999_999, "blocks": [block()] * 1601, "phi_edge_uses": []} + with self.assertRaisesRegex(ValueError, "100 million"): + solve_reference({"version": 1, "functions": [function, function]}) + + def test_seeded_workpacks_match_independent_oracle_and_existing_cpu_layout(self): + total_cells = 0 + for workload in workloads(): + with self.subTest(workpack=workload["name"]): + original = copy.deepcopy(workload) + result = solve_reference(workload) + self.assertEqual(workload, original) + self.assertEqual(result["live_in_words"], oracle_words(workload)) + total_cells += len(result["live_in_words"]) + self.assertEqual(total_cells, 8538) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/gpu-rust-compiler/tests/test_native_adapter.py b/experiments/gpu-rust-compiler/tests/test_native_adapter.py new file mode 100644 index 00000000000..4d0431dee92 --- /dev/null +++ b/experiments/gpu-rust-compiler/tests/test_native_adapter.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Native adapter correctness, persistent GPU reuse, and request isolation.""" +import json +import platform +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BINARY = ROOT / ".build/compiler/release/gpu-rust-compiler" + + +class NativeAdapterTests(unittest.TestCase): + def test_native_backends_match_standard_rust(self): + sources = [ + (ROOT / "fixtures/compiler_demo.rs").read_text(), + """fn danger(n:i64)->bool { return 1/n>0; } + fn main()->i64 { let mut r:i64=17; + if false && danger(0) { r=99; } + if true || danger(0) { r=r+3; } + return r; }""", + """fn main()->bool { let mut i:i64=0; let mut n:i64=0; + while i<9 { if (i&1)==0 { n=n+i; } else { n=n-1; } i=i+1; } + return n==16; }""", + ] + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + directory = Path(directory) + for index, text in enumerate(sources): + source = directory / f"source{index}.rs" + source.write_text(text) + reference = directory / f"reference{index}.rs" + reference.write_text(f'mod p {{ include!("{source}");pub fn run(){{println!("{{}}",main() as i64);}}}}fn main(){{p::run();}}') + reference_bin = directory / f"reference{index}" + built = subprocess.run(["rustc", "-Awarnings", "-C", "overflow-checks=off", str(reference), "-o", str(reference_bin)], capture_output=True, text=True) + self.assertEqual(built.returncode, 0, built.stderr) + expected = subprocess.check_output([str(reference_bin)], text=True) + backends = ("cpu", "metal", "hybrid") if platform.system() == "Darwin" else ("cpu",) + for backend in backends: + output = directory / f"{backend}{index}" + built = subprocess.run([str(BINARY), str(source), "--backend", backend, "--verify", "--output", str(output)], capture_output=True, text=True) + self.assertEqual(built.returncode, 0, built.stderr) + report = json.loads(built.stdout) + self.assertTrue(report["verified_against_cpu"]) + self.assertEqual(subprocess.check_output([str(output)], text=True), expected) + if backend == "metal": + self.assertGreater(report["actual_gpu_functions"], 0) + self.assertGreater(report["gpu_execution_ms"], 0) + else: + self.assertEqual(report["gpu_pipeline_creations"], 0) + + @unittest.skipUnless(platform.system() == "Darwin", "Metal requires macOS") + def test_worker_reuses_pipeline_and_recovers_from_bad_requests(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + directory = Path(directory) + source = directory / "changing.rs" + output = directory / "program" + report_path = directory / "report.json" + process = subprocess.Popen([str(BINARY), "--serve", "--backend", "metal", "--verify"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) + def request(line): + process.stdin.write(line + "\n"); process.stdin.flush() + response = process.stdout.readline() + self.assertTrue(response, "Worker ended without a response") + return json.loads(response) + try: + source.write_text("fn main()->i64 { 18*22; return 7; }") + first = request(f"compile\t{source}\t{output}\t{report_path}") + self.assertEqual(first["status"], "ok") + self.assertEqual(subprocess.check_output([str(output)], text=True), "7\n") + self.assertFalse(first["gpu_pipeline_reused"]) + self.assertGreater(first["gpu_buffer_allocations"], 0) + # Invalid source cannot poison the persistent compiler state. + source.write_text("fn main(){let a=vec![1];}") + invalid = request(f"compile\t{source}\t{output}\t{report_path}") + self.assertEqual(invalid["status"], "error") + self.assertEqual(request("wrong request")["status"], "error") + source.write_text("fn main()->i64 { 18*22; return 19; }") + second = request(f"compile\t{source}\t{output}\t{report_path}") + self.assertEqual(second["status"], "ok") + self.assertEqual(subprocess.check_output([str(output)], text=True), "19\n") + self.assertEqual(first["process_id"], second["process_id"]) + self.assertEqual(second["gpu_pipeline_creations"], 1) + self.assertEqual(second["gpu_submissions_total"], 2) + self.assertTrue(second["gpu_pipeline_reused"]) + self.assertEqual(second["gpu_buffer_allocations"], 0) + self.assertGreater(second["dead_scalar_instructions_removed"], 0) + self.assertEqual(json.loads(report_path.read_text())["request_number"], second["request_number"]) + source.write_text("""fn bump(n:i64)->i64 {let mut y:i64=0;let mut i:i64=0; + while ii64 {return bump(6);}""") + larger = request(f"compile\t{source}\t{output}\t{report_path}") + self.assertEqual(larger["status"], "ok") + self.assertEqual(subprocess.check_output([str(output)], text=True), "12\n") + self.assertEqual(larger["gpu_pipeline_creations"], 1) + self.assertTrue(larger["gpu_pipeline_reused"]) + self.assertGreater(larger["gpu_buffer_allocations"], 0) + source.write_text("fn main()->i64 {return 19;}") + smaller = request(f"compile\t{source}\t{output}\t{report_path}") + self.assertEqual(smaller["status"], "ok") + self.assertEqual(subprocess.check_output([str(output)], text=True), "19\n") + self.assertEqual(smaller["gpu_buffer_allocations"], 0) + self.assertEqual(smaller["gpu_submissions_total"], 4) + finally: + if process.poll() is None: + process.stdin.write("quit\n"); process.stdin.flush() + process.communicate(timeout=20) + self.assertEqual(process.returncode, 0) + + @unittest.skipUnless(platform.system() == "Darwin", "Metal requires macOS") + def test_missing_shader_is_reported_without_gpu_fallback(self): + process = subprocess.run([str(BINARY), str(ROOT / "fixtures/compiler_demo.rs"), "--backend", "metal", + "--shader", str(ROOT / ".build/not-a-shader.metal"), "--output", str(ROOT / ".build/missing-shader-output")], + capture_output=True, text=True) + self.assertNotEqual(process.returncode, 0) + self.assertIn("Could not read Metal shader", process.stderr) + + def test_unsupported_rust_is_rejected_without_emitting_ir(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + directory = Path(directory) + source = directory / "unsupported.rs" + source.write_text("fn main() { let v = vec![1, 2, 3]; }") + output = directory / "output.ll" + process = subprocess.run([str(BINARY), str(source), "--emit-llvm", str(output)], + capture_output=True, text=True) + self.assertNotEqual(process.returncode, 0) + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/gpu-rust-compiler/tests/test_native_cuda_evidence.py b/experiments/gpu-rust-compiler/tests/test_native_cuda_evidence.py new file mode 100644 index 00000000000..48a792b4df4 --- /dev/null +++ b/experiments/gpu-rust-compiler/tests/test_native_cuda_evidence.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Native evidence must reject cached facts and missing actual GPU execution.""" +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from verify_native_cuda import changed_facts, validate_native +from verify_cuda import oracle_words, workloads + + +class NativeCudaEvidenceTests(unittest.TestCase): + def test_changed_inputs_keep_graph_shape_but_cleared_facts_require_zero_results(self): + original = workloads()[0] + expected = oracle_words(original) + changed = changed_facts(original) + cleared = changed_facts(original, erase=True) + self.assertNotEqual(oracle_words(changed), expected) + self.assertEqual(oracle_words(cleared), [0] * len(expected)) + self.assertEqual(original, workloads()[0]) + for old, new in zip(original["functions"], changed["functions"]): + self.assertEqual(old["value_count"], new["value_count"]) + self.assertEqual([row["successors"] for row in old["blocks"]], + [row["successors"] for row in new["blocks"]]) + + def test_gpu_claim_requires_native_context_device_kernel_and_transfers_evidence(self): + response = {"analysis_ms": 1.0, "total_recurrent_ms": 1.2, + "cpu_pool_workers": 8, "actual_gpu_functions": 0} + validate_native(response, [1, 2], [1, 2]) + with self.assertRaisesRegex(ValueError, "CUDA execution"): + validate_native(response, [1, 2], [1, 2], True) + response.update(actual_gpu_functions=3, cuda_context_creations=1, + capabilities={"device_name": "test"}, + gpu_stats={"gpu_ms": 0.2, "resident_input": 0}) + validate_native(response, [1, 2], [1, 2], True) + response["gpu_stats"].update(resident_input=1, host_to_device_ms=0.3, staging_ms=0) + with self.assertRaisesRegex(ValueError, "hidden input transfers"): + validate_native(response, [1, 2], [1, 2], True, True) + response["gpu_stats"]["host_to_device_ms"] = 0 + validate_native(response, [1, 2], [1, 2], True, True) + with self.assertRaisesRegex(ValueError, "oracle"): + validate_native(response, [1, 3], [1, 2], True, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/gpu-rust-compiler/tests/test_prepare_workload.py b/experiments/gpu-rust-compiler/tests/test_prepare_workload.py new file mode 100644 index 00000000000..629eed9d879 --- /dev/null +++ b/experiments/gpu-rust-compiler/tests/test_prepare_workload.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.util +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / 'scripts' / 'prepare_workload.py' +SPEC = importlib.util.spec_from_file_location('prepare_workload', SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def function(ir): + return MODULE.parse_llvm_ir(ir)['functions'][0] + + +def live_in(fn): + """Independent set worklist oracle, preserving edge uses explicitly.""" + incoming = [set() for _ in fn['blocks']] + pending = list(range(len(incoming))) + predecessors = [[] for _ in incoming] + for i, block in enumerate(fn['blocks']): + for succ in block['successors']: + predecessors[succ].append(i) + edges = {} + for edge in fn['phi_edge_uses']: + edges[(edge['from'], edge['to'])] = set(edge['values']) + while pending: + i = pending.pop() + block = fn['blocks'][i] + out = set() + for succ in block['successors']: + out |= incoming[succ] | edges.get((i, succ), set()) + new = set(block['use']) | (out - set(block['def'])) + if new != incoming[i]: + incoming[i] = new + pending.extend(predecessors[i]) + return incoming + + +class ExtractionTests(unittest.TestCase): + def test_implicit_entry_and_return_use(self): + fn = function(''' +define i32 @increment(i32 %0) { + %2 = add i32 %0, 1 + ret i32 %2 +} +''') + self.assertEqual(fn['blocks'], [{'name': '1', 'successors': [], 'use': [0], 'def': [1]}]) + self.assertEqual(live_in(fn), [{0}]) + + def test_phi_is_edge_use_and_predecessor_definition_kills_it(self): + fn = function(''' +define i32 @diamond(i1 %c, i32 %x, i32 %y) { +entry: + br i1 %c, label %left, label %right +left: + %a = add i32 %x, 1 + br label %join +right: + %b = add i32 %y, 2 + br label %join +join: + %p = phi i32 [ %a, %left ], [ %b, %right ] + ret i32 %p +} +''') + self.assertEqual(fn['phi_edge_uses'], [ + {'from': 1, 'to': 3, 'values': [3]}, {'from': 2, 'to': 3, 'values': [4]}]) + self.assertEqual(fn['blocks'][3]['use'], []) + self.assertEqual(live_in(fn), [{0, 1, 2}, {1}, {2}, set()]) + folded = MODULE.fold_phi_edge_uses(fn) + self.assertEqual(folded['blocks'][1]['use'], [1]) + self.assertEqual(folded['blocks'][2]['use'], [2]) + folded['phi_edge_uses'] = [] + self.assertEqual(live_in(folded), live_in(fn)) + + def test_loop_phi_and_backedge(self): + fn = function(''' +define i32 @loop(i32 %n) { +entry: + br label %loop +loop: + %i = phi i32 [ 0, %entry ], [ %next, %loop ] + %next = add i32 %i, 1 + %more = icmp slt i32 %next, %n + br i1 %more, label %loop, label %exit +exit: + ret i32 %next +} +''') + self.assertEqual(fn['blocks'][1]['successors'], [1, 2]) + self.assertEqual(live_in(fn), [{0}, {0}, {2}]) + self.assertEqual(fn['phi_edge_uses'][-1], {'from': 1, 'to': 1, 'values': [2]}) + + def test_array_type_phi_and_nested_constant(self): + fn = function(''' +define [2 x i32] @array_phi(i1 %c, [2 x i32] %arr) { +entry: + br i1 %c, label %left, label %right +left: + br label %join +right: + br label %join +join: + %p = phi [2 x i32] [ %arr, %left ], [ [i32 1, i32 2], %right ] + ret [2 x i32] %p +} +''') + self.assertEqual(fn['phi_edge_uses'], [ + {'from': 1, 'to': 3, 'values': [1]}, {'from': 2, 'to': 3, 'values': []}]) + self.assertEqual(live_in(fn), [{0, 1}, {1}, set(), set()]) + + def test_quoted_names_escaped_semicolon_and_multiline_switch(self): + fn = function(''' +define i32 @"strange;function"(i32 %"x y") { +"first block": + switch i32 %"x y", label %"default\\20block" [ + i32 1, label %"one;block" + ] +"one;block": + ret i32 %"x y" +"default block": + ret i32 0 +} +''') + self.assertEqual(fn['name'], 'strange;function') + self.assertEqual(fn['blocks'][0]['successors'], [2, 1]) + self.assertEqual(fn['blocks'][0]['use'], [0]) + + def test_multiline_invoke_phi_return_value(self): + fn = function(''' +declare i32 @callee(i32) +define i32 @caller(i32 %x) personality ptr @personality { +entry: + %r = invoke i32 @callee(i32 %x) + to label %normal unwind label %unwind +normal: + %p = phi i32 [ %r, %entry ] + ret i32 %p +unwind: + %error = landingpad { ptr, i32 } + cleanup + resume { ptr, i32 } %error +} +''') + self.assertEqual(fn['blocks'][0]['successors'], [1, 2]) + self.assertEqual(live_in(fn), [{0}, set(), set()]) + + def test_named_types_and_debug_records_are_not_runtime_uses(self): + fn = function(''' +%Thing = type { i32 } +define ptr @address(ptr %base, i32 %unused) { +entry: + #dbg_value(i32 %unused, !1, !DIExpression(), !2) + %field = getelementptr %Thing, ptr %base, i32 0, i32 0 + ret ptr %field +} +''') + self.assertEqual(fn['blocks'][0]['use'], [0]) + + def test_fail_closed(self): + cases = [ + 'define void @f() {\nentry:\n newop i32 1\n ret void\n}', + 'define void @f() {\nentry:\n br label %missing\n}', + 'define i32 @f() {\nentry:\n ret i32 %unknown\n}', + 'define void @f() {\nentry:\n call void @g()\n}', + 'define i32 @f() {\nentry:\n ret i32 0\nother:\n %p = phi i32 [ 1, %entry ]\n ret i32 %p\n}', + ] + for ir in cases: + with self.subTest(ir=ir), self.assertRaises(MODULE.IRParseError): + function(ir) + + @unittest.skipUnless(shutil.which('rustc'), 'rustc unavailable') + def test_real_rustc_ir(self): + source = ''' +#[unsafe(no_mangle)] +pub fn branch_loop(xs: &[u32], threshold: u32) -> u32 { + let mut sum = 0u32; + for &x in xs { + if x > threshold { sum = sum.wrapping_add(x); } + else { sum ^= x.rotate_left(3); } + } + sum +} +''' + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / 'fixture.rs').write_text(source) + for level in ['0', '2']: + ir = root / f'fixture_{level}.ll' + subprocess.run(['rustc', '--edition=2024', '--crate-type=lib', '--emit=llvm-ir', + '-C', f'opt-level={level}', '-o', str(ir), str(root / 'fixture.rs')], + check=True, capture_output=True, text=True) + workload = MODULE.parse_llvm_ir(ir.read_text()) + self.assertTrue(workload['functions']) + self.assertGreater(sum(len(f['blocks']) for f in workload['functions']), 1) + for fn in workload['functions']: + folded = MODULE.fold_phi_edge_uses(fn) + folded['phi_edge_uses'] = [] + self.assertEqual(live_in(folded), live_in(fn)) + + +if __name__ == '__main__': + unittest.main()