From d054df07e3b137ff996df74514a2761ded9a160f Mon Sep 17 00:00:00 2001 From: Eric Daniels Date: Mon, 16 Nov 2026 02:44:38 -0800 Subject: [PATCH] nvproxy: Serve /proc/driver/nvidia/version from the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nvproxy now serves the host's /proc/driver/nvidia/version unchanged. runsc reads it where it reads params, before the sandbox enters its chroot, and a host without the file still gets a sandbox without one. Linux has this file whenever the NVIDIA driver is loaded, and libnvidia-container copies it into runc containers, but nvproxy sandboxes have only params. NVIDIA's streaming SDK, which Omniverse Kit and Isaac Sim use for WebRTC livestreaming, reads the version file to pick its NVENC API version. Without it the SDK assumes driver 410.96, picks NVENC API 8, and every session fails with NVST_DISCONN_SERVER_VIDEO_ENCODER_INIT_FAILED. Tested: nvconf and nvproxy unit tests on master. The same change on a release branch, on a T4 with driver 610.57.04, served the host's file and let Isaac Sim 6.2's WebRTC livestream stream 1080p60 to Chrome, which fails with the encoder init error without it. The new GPU test, TestProcDriverNvidiaVersion in test/gpu:procfs_test, has not run on a GPU host yet. Auto-fixed-by: [🤖] @EtiennePerot FUTURE_COPYBARA_INTEGRATE_REVIEW=https://github.com/google/gvisor/pull/17497 from edaniels:nvproxy-driver-version 0c0975a3fefc5880bf5e8fd5c7e9227f0cddef84 PiperOrigin-RevId: 1022654275 --- Makefile | 2 + pkg/sentry/devices/nvproxy/BUILD | 7 +- pkg/sentry/devices/nvproxy/nvconf/BUILD | 5 +- .../devices/nvproxy/nvconf/hostsettings.go | 48 ++++++-- .../nvproxy/nvconf/hostsettings_test.go | 112 ++++++++++++++++++ pkg/sentry/devices/nvproxy/nvproxy.go | 30 ++--- pkg/sentry/devices/nvproxy/procfs.go | 6 + pkg/sentry/devices/nvproxy/procfs_test.go | 77 ++++++++++++ runsc/cmd/sentry/sentrycmd/BUILD | 7 +- runsc/cmd/sentry/sentrycmd/boot.go | 103 +++++++++------- runsc/cmd/sentry/sentrycmd/boot_test.go | 73 ++++++++++++ test/gpu/BUILD | 14 +++ test/gpu/procfs_test.go | 52 ++++++++ 13 files changed, 466 insertions(+), 70 deletions(-) create mode 100644 pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go create mode 100644 pkg/sentry/devices/nvproxy/procfs_test.go create mode 100644 runsc/cmd/sentry/sentrycmd/boot_test.go create mode 100644 test/gpu/procfs_test.go diff --git a/Makefile b/Makefile index aec4810df95..21ced535595 100644 --- a/Makefile +++ b/Makefile @@ -408,6 +408,7 @@ l4-gpu-tests: l4-gpu-images $(RUNTIME_BIN) gpu-all-tests: gpu-images gpu-smoke-tests $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME),--nvproxy=true --nvproxy-allowed-driver-capabilities=all) + @$(call sudo,test/gpu:procfs_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @$(call sudo,test/gpu:pytorch_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @$(call sudo,test/gpu:textgen_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @$(call sudo,test/gpu:imagegen_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @@ -420,6 +421,7 @@ gpu-all-tests: gpu-images gpu-smoke-tests $(RUNTIME_BIN) cos-gpu-all-tests: gpu-images cos-gpu-smoke-tests $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME),--nvproxy=true) + @$(call sudo,test/gpu:procfs_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) @$(call sudo,test/gpu:pytorch_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) @$(call sudo,test/gpu:textgen_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) @$(call sudo,test/gpu:imagegen_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index d8c09dafbe5..c04012b79ee 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -103,12 +103,17 @@ go_library( go_test( name = "nvproxy_test", - srcs = ["nvproxy_test.go"], + srcs = [ + "nvproxy_test.go", + "procfs_test.go", + ], embed = [":nvproxy"], deps = [ "//pkg/abi/nvgpu", "//pkg/seccomp", + "//pkg/sentry/contexttest", "//pkg/sentry/devices/nvproxy/nvconf", + "//pkg/sentry/vfs", "@org_golang_x_sys//unix:go_default_library", ], ) diff --git a/pkg/sentry/devices/nvproxy/nvconf/BUILD b/pkg/sentry/devices/nvproxy/nvconf/BUILD index 04b4bf7ad42..516834141b4 100644 --- a/pkg/sentry/devices/nvproxy/nvconf/BUILD +++ b/pkg/sentry/devices/nvproxy/nvconf/BUILD @@ -29,6 +29,9 @@ go_library( go_test( name = "nvconf_test", size = "small", - srcs = ["caps_test.go"], + srcs = [ + "caps_test.go", + "hostsettings_test.go", + ], embed = [":nvconf"], ) diff --git a/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go b/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go index f52c56abb93..04d6dcd78a2 100644 --- a/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go +++ b/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "regexp" "strconv" @@ -30,6 +31,10 @@ type HostSettings struct { // ProcDriverNvidiaParams is the contents of /proc/driver/nvidia/params. ProcDriverNvidiaParams string + // ProcDriverNvidiaVersion is the contents of /proc/driver/nvidia/version, + // or empty if the host has no such file or it cannot be read. + ProcDriverNvidiaVersion string + // If HaveFabricIMEXManagement is true, FabricIMEXManagementDevMinor is the // device minor number advertised in // /proc/driver/nvidia/capabilities/fabric-imex-mgmt. @@ -52,23 +57,36 @@ type HostSettingsOptions struct { // GetHostSettings returns HostSettings. func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) { + return getHostSettingsFrom("/proc/driver/nvidia", opts) +} + +func getHostSettingsFrom(nvidiaProcDir string, opts HostSettingsOptions) (*HostSettings, error) { settings := &HostSettings{} - params, err := os.ReadFile("/proc/driver/nvidia/params") + paramsPath := filepath.Join(nvidiaProcDir, "params") + params, err := os.ReadFile(paramsPath) if err != nil { - return nil, fmt.Errorf("failed to read /proc/driver/nvidia/params: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", paramsPath, err) } settings.ProcDriverNvidiaParams = string(params) + versionPath := filepath.Join(nvidiaProcDir, "version") + version, err := readFileIfExists(versionPath) + if err != nil { + log.Warningf("Not serving %s: %v", versionPath, err) + } + settings.ProcDriverNvidiaVersion = version + if opts.WantFabricIMEXManagement { - fabricImexMgmt, err := os.ReadFile("/proc/driver/nvidia/capabilities/fabric-imex-mgmt") + fabricImexPath := filepath.Join(nvidiaProcDir, "capabilities", "fabric-imex-mgmt") + fabricImexMgmt, err := os.ReadFile(fabricImexPath) if err != nil { - return nil, fmt.Errorf("failed to read /proc/driver/nvidia/capabilities/fabric-imex-mgmt: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", fabricImexPath, err) } minorRe := regexp.MustCompile(`DeviceFileMinor: (\d+)`) m := minorRe.FindSubmatch(fabricImexMgmt) if m == nil { - return nil, fmt.Errorf("failed to find DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt") + return nil, fmt.Errorf("failed to find DeviceFileMinor in %s", fabricImexPath) } minor, err := strconv.ParseUint(string(m[1]), 10, 32) if err != nil { @@ -77,10 +95,11 @@ func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) { settings.HaveFabricIMEXManagement = true settings.FabricIMEXManagementDevMinor = uint32(minor) - if mnccMgmt, err := os.ReadFile("/proc/driver/nvidia/capabilities/mncc-mgmt"); err == nil { + mnccPath := filepath.Join(nvidiaProcDir, "capabilities", "mncc-mgmt") + if mnccMgmt, err := os.ReadFile(mnccPath); err == nil { m := minorRe.FindSubmatch(mnccMgmt) if m == nil { - return nil, fmt.Errorf("failed to find DeviceFileMinor in /proc/driver/nvidia/capabilities/mncc-mgmt") + return nil, fmt.Errorf("failed to find DeviceFileMinor in %s", mnccPath) } mnccMinor, err := strconv.ParseUint(string(m[1]), 10, 32) if err != nil { @@ -89,13 +108,26 @@ func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) { settings.HaveMNCCManagement = true settings.MNCCManagementDevMinor = uint32(mnccMinor) } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("failed to read /proc/driver/nvidia/capabilities/mncc-mgmt: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", mnccPath, err) } } return settings, nil } +// readFileIfExists returns the contents of the file at path, or an empty +// string if no file exists at path. +func readFileIfExists(path string) (string, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", path, err) + } + return string(data), nil +} + // IMEXChannelCount returns the number of IMEX channels indicated by // /proc/driver/nvidia/params. See description of NVreg_ImexChannelCount in the // Nvidia GPU driver's kernel-open/nvidia/nv-reg.h. diff --git a/pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go b/pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go new file mode 100644 index 00000000000..491bbe02e17 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go @@ -0,0 +1,112 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvconf + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetHostSettingsFrom(t *testing.T) { + const wantParams = "ModifyDeviceFiles: 1\nImexChannelCount: 4\n" + const wantVersion = "NVRM version: NVIDIA UNIX x86_64 Kernel Module 610.57.04\n" + + for _, tc := range []struct { + name string + setup func(t *testing.T, dir string) + wantVersion string + }{ + { + name: "with_version", + setup: func(t *testing.T, dir string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, "version"), []byte(wantVersion), 0644); err != nil { + t.Fatalf("os.WriteFile(version) failed: %v", err) + } + }, + wantVersion: wantVersion, + }, + { + name: "without_version", + setup: func(t *testing.T, dir string) {}, + wantVersion: "", + }, + { + name: "unreadable_version", + setup: func(t *testing.T, dir string) { + t.Helper() + if err := os.Mkdir(filepath.Join(dir, "version"), 0755); err != nil { + t.Fatalf("os.Mkdir(version) failed: %v", err) + } + }, + wantVersion: "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "params"), []byte(wantParams), 0644); err != nil { + t.Fatalf("os.WriteFile(params) failed: %v", err) + } + tc.setup(t, dir) + + got, err := getHostSettingsFrom(dir, HostSettingsOptions{}) + if err != nil { + t.Fatalf("getHostSettingsFrom() failed: %v", err) + } + if got.ProcDriverNvidiaParams != wantParams { + t.Errorf("ProcDriverNvidiaParams = %q, want %q", got.ProcDriverNvidiaParams, wantParams) + } + if got.ProcDriverNvidiaVersion != tc.wantVersion { + t.Errorf("ProcDriverNvidiaVersion = %q, want %q", got.ProcDriverNvidiaVersion, tc.wantVersion) + } + }) + } +} + +func TestReadFileIfExists(t *testing.T) { + dir := t.TempDir() + existing := filepath.Join(dir, "existing") + const want = "hello\n" + if err := os.WriteFile(existing, []byte(want), 0644); err != nil { + t.Fatalf("os.WriteFile() failed: %v", err) + } + + got, err := readFileIfExists(existing) + if err != nil { + t.Fatalf("readFileIfExists(%q) failed: %v", existing, err) + } + if got != want { + t.Errorf("readFileIfExists(%q) = %q, want %q", existing, got, want) + } + + missing := filepath.Join(dir, "missing") + got, err = readFileIfExists(missing) + if err != nil { + t.Fatalf("readFileIfExists(%q) failed: %v", missing, err) + } + if got != "" { + t.Errorf("readFileIfExists(%q) = %q, want empty", missing, got) + } + + subdir := filepath.Join(dir, "subdir") + if err := os.Mkdir(subdir, 0755); err != nil { + t.Fatalf("os.Mkdir() failed: %v", err) + } + got, err = readFileIfExists(subdir) + if err == nil { + t.Errorf("readFileIfExists(%q) succeeded with %q, want error", subdir, got) + } +} diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go index 4990e36cd3d..a11e95f631a 100644 --- a/pkg/sentry/devices/nvproxy/nvproxy.go +++ b/pkg/sentry/devices/nvproxy/nvproxy.go @@ -81,13 +81,14 @@ func Register(vfsObj *vfs.VirtualFilesystem, opts *Options) (*DeviceInfo, error) log.Warningf("nvproxy: NVIDIA driver capability set is empty; all GPU operations will fail") } nvp := &nvproxy{ - abi: abiEntry.cons(), - version: opts.DriverVersion, - capsEnabled: opts.DriverCaps, - useDevGofer: opts.UseDevGofer, - procDriverNvidiaParams: opts.HostSettings.ProcDriverNvidiaParams, - frontendFDs: make(map[*frontendFD]struct{}), - clients: make(map[nvgpu.Handle]*rootClient), + abi: abiEntry.cons(), + version: opts.DriverVersion, + capsEnabled: opts.DriverCaps, + useDevGofer: opts.UseDevGofer, + procDriverNvidiaParams: opts.HostSettings.ProcDriverNvidiaParams, + procDriverNvidiaVersion: opts.HostSettings.ProcDriverNvidiaVersion, + frontendFDs: make(map[*frontendFD]struct{}), + clients: make(map[nvgpu.Handle]*rootClient), } // Force ModifyDeviceFiles in /proc/driver/nvidia/params to 0. This is // consistent with libnvidia-container's src/nvc_mount.c:mount_procfs(). @@ -237,13 +238,14 @@ func DeviceInfoFromVFS(vfsObj *vfs.VirtualFilesystem) *DeviceInfo { // +stateify savable type nvproxy struct { - abi *driverABI `state:"nosave"` - version nvconf.DriverVersion - capsEnabled nvconf.DriverCaps - useDevGofer bool - procDriverNvidiaParams string - devInfo DeviceInfo - regularDevs [nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX + 1]*frontendDevice + abi *driverABI `state:"nosave"` + version nvconf.DriverVersion + capsEnabled nvconf.DriverCaps + useDevGofer bool + procDriverNvidiaParams string + procDriverNvidiaVersion string + devInfo DeviceInfo + regularDevs [nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX + 1]*frontendDevice fdsMu fdsMutex `state:"nosave"` frontendFDs map[*frontendFD]struct{} diff --git a/pkg/sentry/devices/nvproxy/procfs.go b/pkg/sentry/devices/nvproxy/procfs.go index 12dc03c332e..059ecda0c0f 100644 --- a/pkg/sentry/devices/nvproxy/procfs.go +++ b/pkg/sentry/devices/nvproxy/procfs.go @@ -40,6 +40,12 @@ func ProcfsInfoFromVFS(vfsObj *vfs.VirtualFilesystem) *ProcfsInfo { "params": nvp.procDriverNvidiaParams, }, } + // The version file exists only if the host has one, and holds the host's + // contents unchanged, as mount_procfs() in libnvidia-container's + // src/nvc_mount.c copies it. + if nvp.procDriverNvidiaVersion != "" { + procfsInfo.StaticFiles["version"] = nvp.procDriverNvidiaVersion + } if nvp.devInfo.HaveFabricIMEXManagement { procfsInfo.StaticFiles["capabilities/fabric-imex-mgmt"] = procfsCapability(nvp.devInfo.FabricIMEXManagementDevMinor, 0o400) } diff --git a/pkg/sentry/devices/nvproxy/procfs_test.go b/pkg/sentry/devices/nvproxy/procfs_test.go new file mode 100644 index 00000000000..1e1cfe41620 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/procfs_test.go @@ -0,0 +1,77 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "testing" + + "gvisor.dev/gvisor/pkg/sentry/contexttest" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// TestProcfsVersionIsHostVersion checks that /proc/driver/nvidia/version +// holds the host's file verbatim, and does not exist when the host has none. +func TestProcfsVersionIsHostVersion(t *testing.T) { + const hostVersion = "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 610.57.04 Release Build (dvs-builder@U22-I3-AF03-09-1) Thu Oct 2 17:07:40 UTC 2026\nGCC version: gcc version 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04)\n" + Init() + var driverVersion nvconf.DriverVersion + for version := range abis { + driverVersion = version + break + } + for _, tc := range []struct { + name string + hostVersion string + wantExists bool + }{ + { + name: "host_has_version", + hostVersion: hostVersion, + wantExists: true, + }, + { + name: "host_has_no_version", + hostVersion: "", + wantExists: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := contexttest.Context(t) + vfsObj := &vfs.VirtualFilesystem{} + if err := vfsObj.Init(ctx); err != nil { + t.Fatalf("vfsObj.Init() failed: %v", err) + } + if _, err := Register(vfsObj, &Options{ + DriverVersion: driverVersion, + AllowUnsupportedDriver: true, + DriverCaps: nvconf.CapCompute | nvconf.CapUtility, + HostSettings: &nvconf.HostSettings{ + ProcDriverNvidiaParams: "ModifyDeviceFiles: 1\n", + ProcDriverNvidiaVersion: tc.hostVersion, + }, + }); err != nil { + t.Fatalf("Register() failed: %v", err) + } + got, exists := ProcfsInfoFromVFS(vfsObj).StaticFiles["version"] + if exists != tc.wantExists { + t.Fatalf("version exists = %t, want %t", exists, tc.wantExists) + } + if got != tc.hostVersion { + t.Errorf("version = %q, want %q", got, tc.hostVersion) + } + }) + } +} diff --git a/runsc/cmd/sentry/sentrycmd/BUILD b/runsc/cmd/sentry/sentrycmd/BUILD index 7bb749aaf7a..cc77bbba5a8 100644 --- a/runsc/cmd/sentry/sentrycmd/BUILD +++ b/runsc/cmd/sentry/sentrycmd/BUILD @@ -56,10 +56,15 @@ go_library( go_test( name = "sentrycmd_test", size = "small", - srcs = ["chroot_test.go"], + srcs = [ + "boot_test.go", + "chroot_test.go", + ], embed = [":sentrycmd"], deps = [ + "//pkg/sentry/devices/nvproxy/nvconf", "//runsc/config", + "//runsc/flag", "//runsc/specutils", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", "@org_golang_x_sys//unix:go_default_library", diff --git a/runsc/cmd/sentry/sentrycmd/boot.go b/runsc/cmd/sentry/sentrycmd/boot.go index a0b1511afa2..1600688017f 100644 --- a/runsc/cmd/sentry/sentrycmd/boot.go +++ b/runsc/cmd/sentry/sentrycmd/boot.go @@ -235,6 +235,7 @@ type Boot struct { // here because nvconf.HostSettings.FabricIMEXManagementDevMinor in // particular is uint32 and there is no flag.Uint32Var(). procDriverNvidiaParams string + procDriverNvidiaVersion string nvidiaFabricIMEXManagementDevMinor int64 nvidiaMNCCManagementDevMinor int64 @@ -328,10 +329,37 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { // Nvidia driver properties. f.StringVar(&b.nvidiaDriverVersion, "nvidia-driver-version", "", "Nvidia driver ABI version to use, or 'latest' for the newest supported one") f.StringVar(&b.procDriverNvidiaParams, "nvidia-host-params", "", "value of /proc/driver/nvidia/params on the host") + f.StringVar(&b.procDriverNvidiaVersion, "nvidia-host-version", "", "value of /proc/driver/nvidia/version on the host, or empty if the host has none") f.Int64Var(&b.nvidiaFabricIMEXManagementDevMinor, "nvidia-fabric-imex-mgmt-minor", -1, "DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt on the host") f.Int64Var(&b.nvidiaMNCCManagementDevMinor, "nvidia-mncc-mgmt-minor", -1, "DeviceFileMinor in /proc/driver/nvidia/capabilities/mncc-mgmt on the host") } +func (b *Boot) applyNvidiaHostSettings(nvhs *nvconf.HostSettings, argOverride map[string]string) { + b.procDriverNvidiaParams = nvhs.ProcDriverNvidiaParams + argOverride["nvidia-host-params"] = nvhs.ProcDriverNvidiaParams + b.procDriverNvidiaVersion = nvhs.ProcDriverNvidiaVersion + argOverride["nvidia-host-version"] = nvhs.ProcDriverNvidiaVersion + if nvhs.HaveFabricIMEXManagement { + b.nvidiaFabricIMEXManagementDevMinor = int64(nvhs.FabricIMEXManagementDevMinor) + argOverride["nvidia-fabric-imex-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.FabricIMEXManagementDevMinor), 10) + } + if nvhs.HaveMNCCManagement { + b.nvidiaMNCCManagementDevMinor = int64(nvhs.MNCCManagementDevMinor) + argOverride["nvidia-mncc-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.MNCCManagementDevMinor), 10) + } +} + +func (b *Boot) nvidiaHostSettings() *nvconf.HostSettings { + return &nvconf.HostSettings{ + ProcDriverNvidiaParams: b.procDriverNvidiaParams, + ProcDriverNvidiaVersion: b.procDriverNvidiaVersion, + HaveFabricIMEXManagement: b.nvidiaFabricIMEXManagementDevMinor >= 0, + FabricIMEXManagementDevMinor: uint32(b.nvidiaFabricIMEXManagementDevMinor), + HaveMNCCManagement: b.nvidiaMNCCManagementDevMinor >= 0, + MNCCManagementDevMinor: uint32(b.nvidiaMNCCManagementDevMinor), + } +} + // willReexec returns true if this boot process will re-execute itself to // apply credential/capability changes to all of its threads, rather than // applying capabilities to all threads in-process. @@ -462,16 +490,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma if err != nil { log.Warningf("Failed to get nvconf.HostSettings: %v", err) } else { - b.procDriverNvidiaParams = nvhs.ProcDriverNvidiaParams - argOverride["nvidia-host-params"] = nvhs.ProcDriverNvidiaParams - if nvhs.HaveFabricIMEXManagement { - b.nvidiaFabricIMEXManagementDevMinor = int64(nvhs.FabricIMEXManagementDevMinor) - argOverride["nvidia-fabric-imex-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.FabricIMEXManagementDevMinor), 10) - } - if nvhs.HaveMNCCManagement { - b.nvidiaMNCCManagementDevMinor = int64(nvhs.MNCCManagementDevMinor) - argOverride["nvidia-mncc-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.MNCCManagementDevMinor), 10) - } + b.applyNvidiaHostSettings(nvhs, argOverride) } } timer.Reached("set nvproxy flags") @@ -718,41 +737,35 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma // Create the loader. bootArgs := boot.Args{ - ID: f.Arg(0), - Spec: spec, - Conf: conf, - ControllerFD: b.controllerFD, - Device: fd.New(b.deviceFD), - PinRingFD: b.pinRingFD, - CPUDMALatencyFD: b.cpuDMALatencyFD, - GoferFDs: b.ioFDs.GetArray(), - DevGoferFD: b.devIoFD, - StdioFDs: b.stdioFDs.GetArray(), - PassFDs: b.passFDs.GetArray(), - ExecFD: b.execFD, - ProcSelfFD: procSelfFD, - GoferFilestoreFDs: b.goferFilestoreFDs.GetArray(), - GoferMountConfs: b.goferMountConfs.GetArray(), - NumCPU: b.cpuNum, - LifecycleNumCPU: b.lifecycleCPUNum, - CPUQuota: b.cpuQuota, - CPUPeriod: b.cpuPeriod, - TotalMem: b.totalMem, - TotalHostMem: b.totalHostMem, - UserLogFD: b.userLogFD, - RDMASysfs: rdmaSnap, - ProductName: b.productName, - PodInitConfigFD: b.podInitConfigFD, - SinkFDs: b.sinkFDs.GetArray(), - ProfileOpts: profile.MakeOpts(&b.profileFDs, conf.ProfileGCInterval), - NvidiaDriverVersion: nvidiaDriverVersion, - NvidiaHostSettings: &nvconf.HostSettings{ - ProcDriverNvidiaParams: b.procDriverNvidiaParams, - HaveFabricIMEXManagement: b.nvidiaFabricIMEXManagementDevMinor >= 0, - FabricIMEXManagementDevMinor: uint32(b.nvidiaFabricIMEXManagementDevMinor), - HaveMNCCManagement: b.nvidiaMNCCManagementDevMinor >= 0, - MNCCManagementDevMinor: uint32(b.nvidiaMNCCManagementDevMinor), - }, + ID: f.Arg(0), + Spec: spec, + Conf: conf, + ControllerFD: b.controllerFD, + Device: fd.New(b.deviceFD), + PinRingFD: b.pinRingFD, + CPUDMALatencyFD: b.cpuDMALatencyFD, + GoferFDs: b.ioFDs.GetArray(), + DevGoferFD: b.devIoFD, + StdioFDs: b.stdioFDs.GetArray(), + PassFDs: b.passFDs.GetArray(), + ExecFD: b.execFD, + ProcSelfFD: procSelfFD, + GoferFilestoreFDs: b.goferFilestoreFDs.GetArray(), + GoferMountConfs: b.goferMountConfs.GetArray(), + NumCPU: b.cpuNum, + LifecycleNumCPU: b.lifecycleCPUNum, + CPUQuota: b.cpuQuota, + CPUPeriod: b.cpuPeriod, + TotalMem: b.totalMem, + TotalHostMem: b.totalHostMem, + UserLogFD: b.userLogFD, + RDMASysfs: rdmaSnap, + ProductName: b.productName, + PodInitConfigFD: b.podInitConfigFD, + SinkFDs: b.sinkFDs.GetArray(), + ProfileOpts: profile.MakeOpts(&b.profileFDs, conf.ProfileGCInterval), + NvidiaDriverVersion: nvidiaDriverVersion, + NvidiaHostSettings: b.nvidiaHostSettings(), HostTHP: b.hostTHP, SaveFDs: b.saveFDs.GetFDs(), SaveCheckpointGofer: b.saveCheckpointGofer, diff --git a/runsc/cmd/sentry/sentrycmd/boot_test.go b/runsc/cmd/sentry/sentrycmd/boot_test.go new file mode 100644 index 00000000000..a485ba8f567 --- /dev/null +++ b/runsc/cmd/sentry/sentrycmd/boot_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sentrycmd + +import ( + "testing" + + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/runsc/flag" +) + +func TestBootNvidiaHostVersionFlag(t *testing.T) { + const wantVersion = "NVRM version: NVIDIA UNIX x86_64 Kernel Module 610.57.04\n" + b := &Boot{} + f := flag.NewFlagSet("test", flag.ContinueOnError) + b.SetFlags(f) + if err := f.Parse([]string{"--nvidia-host-version=" + wantVersion}); err != nil { + t.Fatalf("f.Parse() failed: %v", err) + } + if b.procDriverNvidiaVersion != wantVersion { + t.Errorf("procDriverNvidiaVersion = %q, want %q", b.procDriverNvidiaVersion, wantVersion) + } + if got := b.nvidiaHostSettings().ProcDriverNvidiaVersion; got != wantVersion { + t.Errorf("nvidiaHostSettings().ProcDriverNvidiaVersion = %q, want %q", got, wantVersion) + } +} + +func TestBootApplyNvidiaHostSettings(t *testing.T) { + b := &Boot{ + nvidiaFabricIMEXManagementDevMinor: -1, + nvidiaMNCCManagementDevMinor: -1, + } + argOverride := make(map[string]string) + want := &nvconf.HostSettings{ + ProcDriverNvidiaParams: "ModifyDeviceFiles: 1\n", + ProcDriverNvidiaVersion: "NVRM version: 610.57.04\n", + HaveFabricIMEXManagement: true, + FabricIMEXManagementDevMinor: 3, + HaveMNCCManagement: true, + MNCCManagementDevMinor: 4, + } + b.applyNvidiaHostSettings(want, argOverride) + + if argOverride["nvidia-host-params"] != want.ProcDriverNvidiaParams { + t.Errorf("argOverride[nvidia-host-params] = %q, want %q", argOverride["nvidia-host-params"], want.ProcDriverNvidiaParams) + } + if argOverride["nvidia-host-version"] != want.ProcDriverNvidiaVersion { + t.Errorf("argOverride[nvidia-host-version] = %q, want %q", argOverride["nvidia-host-version"], want.ProcDriverNvidiaVersion) + } + if argOverride["nvidia-fabric-imex-mgmt-minor"] != "3" { + t.Errorf("argOverride[nvidia-fabric-imex-mgmt-minor] = %q, want %q", argOverride["nvidia-fabric-imex-mgmt-minor"], "3") + } + if argOverride["nvidia-mncc-mgmt-minor"] != "4" { + t.Errorf("argOverride[nvidia-mncc-mgmt-minor] = %q, want %q", argOverride["nvidia-mncc-mgmt-minor"], "4") + } + + got := b.nvidiaHostSettings() + if *got != *want { + t.Errorf("nvidiaHostSettings() = %+v, want %+v", *got, *want) + } +} diff --git a/test/gpu/BUILD b/test/gpu/BUILD index cffaad91e43..0c6a3de37da 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -19,6 +19,20 @@ go_test( deps = ["//pkg/test/dockerutil"], ) +go_test( + name = "procfs_test", + srcs = ["procfs_test.go"], + # Needed to invalidate the bazel cache in case of any code changes. + data = ["//:release"], + tags = [ + "manual", + "noguitar", + "notap", + ], + visibility = ["//:sandbox"], + deps = ["//pkg/test/dockerutil"], +) + go_test( name = "pytorch_test", srcs = ["pytorch_test.go"], diff --git a/test/gpu/procfs_test.go b/test/gpu/procfs_test.go new file mode 100644 index 00000000000..9821c8a1e6f --- /dev/null +++ b/test/gpu/procfs_test.go @@ -0,0 +1,52 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package procfs_test tests the NVIDIA driver files in /proc/driver/nvidia. +package procfs_test + +import ( + "context" + "os" + "testing" + + "gvisor.dev/gvisor/pkg/test/dockerutil" +) + +// TestProcDriverNvidiaVersion checks that /proc/driver/nvidia/version in the +// container is the host's file verbatim. +func TestProcDriverNvidiaVersion(t *testing.T) { + want, err := os.ReadFile("/proc/driver/nvidia/version") + if err != nil { + t.Fatalf("failed to read the host's /proc/driver/nvidia/version: %v", err) + } + ctx := context.Background() + c := dockerutil.MakeContainer(ctx, t) + defer c.CleanUp(ctx) + + opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + Capabilities: "all", + DisableSnifferReason: "reads a procfs file and makes no ioctls", + }) + if err != nil { + t.Fatalf("failed to get GPU run options: %v", err) + } + opts.Image = "gpu/cuda-tests" + got, err := c.Run(ctx, opts, "cat", "/proc/driver/nvidia/version") + if err != nil { + t.Fatalf("could not read /proc/driver/nvidia/version: %v", err) + } + if got != string(want) { + t.Errorf("/proc/driver/nvidia/version = %q, want %q", got, string(want)) + } +}