diff --git a/Makefile b/Makefile index aec4810df95..21ced535595 100644 --- a/Makefile +++ b/Makefile @@ -408,6 +408,7 @@ l4-gpu-tests: l4-gpu-images $(RUNTIME_BIN) gpu-all-tests: gpu-images gpu-smoke-tests $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME),--nvproxy=true --nvproxy-allowed-driver-capabilities=all) + @$(call sudo,test/gpu:procfs_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @$(call sudo,test/gpu:pytorch_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @$(call sudo,test/gpu:textgen_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @$(call sudo,test/gpu:imagegen_test,--runtime=$(RUNTIME) -test.v $(ARGS)) @@ -420,6 +421,7 @@ gpu-all-tests: gpu-images gpu-smoke-tests $(RUNTIME_BIN) cos-gpu-all-tests: gpu-images cos-gpu-smoke-tests $(RUNTIME_BIN) @$(call install_runtime,$(RUNTIME),--nvproxy=true) + @$(call sudo,test/gpu:procfs_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) @$(call sudo,test/gpu:pytorch_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) @$(call sudo,test/gpu:textgen_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) @$(call sudo,test/gpu:imagegen_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS)) diff --git a/pkg/sentry/devices/nvproxy/BUILD b/pkg/sentry/devices/nvproxy/BUILD index d8c09dafbe5..c04012b79ee 100644 --- a/pkg/sentry/devices/nvproxy/BUILD +++ b/pkg/sentry/devices/nvproxy/BUILD @@ -103,12 +103,17 @@ go_library( go_test( name = "nvproxy_test", - srcs = ["nvproxy_test.go"], + srcs = [ + "nvproxy_test.go", + "procfs_test.go", + ], embed = [":nvproxy"], deps = [ "//pkg/abi/nvgpu", "//pkg/seccomp", + "//pkg/sentry/contexttest", "//pkg/sentry/devices/nvproxy/nvconf", + "//pkg/sentry/vfs", "@org_golang_x_sys//unix:go_default_library", ], ) diff --git a/pkg/sentry/devices/nvproxy/nvconf/BUILD b/pkg/sentry/devices/nvproxy/nvconf/BUILD index 04b4bf7ad42..516834141b4 100644 --- a/pkg/sentry/devices/nvproxy/nvconf/BUILD +++ b/pkg/sentry/devices/nvproxy/nvconf/BUILD @@ -29,6 +29,9 @@ go_library( go_test( name = "nvconf_test", size = "small", - srcs = ["caps_test.go"], + srcs = [ + "caps_test.go", + "hostsettings_test.go", + ], embed = [":nvconf"], ) diff --git a/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go b/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go index f52c56abb93..04d6dcd78a2 100644 --- a/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go +++ b/pkg/sentry/devices/nvproxy/nvconf/hostsettings.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "regexp" "strconv" @@ -30,6 +31,10 @@ type HostSettings struct { // ProcDriverNvidiaParams is the contents of /proc/driver/nvidia/params. ProcDriverNvidiaParams string + // ProcDriverNvidiaVersion is the contents of /proc/driver/nvidia/version, + // or empty if the host has no such file or it cannot be read. + ProcDriverNvidiaVersion string + // If HaveFabricIMEXManagement is true, FabricIMEXManagementDevMinor is the // device minor number advertised in // /proc/driver/nvidia/capabilities/fabric-imex-mgmt. @@ -52,23 +57,36 @@ type HostSettingsOptions struct { // GetHostSettings returns HostSettings. func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) { + return getHostSettingsFrom("/proc/driver/nvidia", opts) +} + +func getHostSettingsFrom(nvidiaProcDir string, opts HostSettingsOptions) (*HostSettings, error) { settings := &HostSettings{} - params, err := os.ReadFile("/proc/driver/nvidia/params") + paramsPath := filepath.Join(nvidiaProcDir, "params") + params, err := os.ReadFile(paramsPath) if err != nil { - return nil, fmt.Errorf("failed to read /proc/driver/nvidia/params: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", paramsPath, err) } settings.ProcDriverNvidiaParams = string(params) + versionPath := filepath.Join(nvidiaProcDir, "version") + version, err := readFileIfExists(versionPath) + if err != nil { + log.Warningf("Not serving %s: %v", versionPath, err) + } + settings.ProcDriverNvidiaVersion = version + if opts.WantFabricIMEXManagement { - fabricImexMgmt, err := os.ReadFile("/proc/driver/nvidia/capabilities/fabric-imex-mgmt") + fabricImexPath := filepath.Join(nvidiaProcDir, "capabilities", "fabric-imex-mgmt") + fabricImexMgmt, err := os.ReadFile(fabricImexPath) if err != nil { - return nil, fmt.Errorf("failed to read /proc/driver/nvidia/capabilities/fabric-imex-mgmt: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", fabricImexPath, err) } minorRe := regexp.MustCompile(`DeviceFileMinor: (\d+)`) m := minorRe.FindSubmatch(fabricImexMgmt) if m == nil { - return nil, fmt.Errorf("failed to find DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt") + return nil, fmt.Errorf("failed to find DeviceFileMinor in %s", fabricImexPath) } minor, err := strconv.ParseUint(string(m[1]), 10, 32) if err != nil { @@ -77,10 +95,11 @@ func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) { settings.HaveFabricIMEXManagement = true settings.FabricIMEXManagementDevMinor = uint32(minor) - if mnccMgmt, err := os.ReadFile("/proc/driver/nvidia/capabilities/mncc-mgmt"); err == nil { + mnccPath := filepath.Join(nvidiaProcDir, "capabilities", "mncc-mgmt") + if mnccMgmt, err := os.ReadFile(mnccPath); err == nil { m := minorRe.FindSubmatch(mnccMgmt) if m == nil { - return nil, fmt.Errorf("failed to find DeviceFileMinor in /proc/driver/nvidia/capabilities/mncc-mgmt") + return nil, fmt.Errorf("failed to find DeviceFileMinor in %s", mnccPath) } mnccMinor, err := strconv.ParseUint(string(m[1]), 10, 32) if err != nil { @@ -89,13 +108,26 @@ func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) { settings.HaveMNCCManagement = true settings.MNCCManagementDevMinor = uint32(mnccMinor) } else if !errors.Is(err, os.ErrNotExist) { - return nil, fmt.Errorf("failed to read /proc/driver/nvidia/capabilities/mncc-mgmt: %w", err) + return nil, fmt.Errorf("failed to read %s: %w", mnccPath, err) } } return settings, nil } +// readFileIfExists returns the contents of the file at path, or an empty +// string if no file exists at path. +func readFileIfExists(path string) (string, error) { + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", path, err) + } + return string(data), nil +} + // IMEXChannelCount returns the number of IMEX channels indicated by // /proc/driver/nvidia/params. See description of NVreg_ImexChannelCount in the // Nvidia GPU driver's kernel-open/nvidia/nv-reg.h. diff --git a/pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go b/pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go new file mode 100644 index 00000000000..491bbe02e17 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go @@ -0,0 +1,112 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvconf + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetHostSettingsFrom(t *testing.T) { + const wantParams = "ModifyDeviceFiles: 1\nImexChannelCount: 4\n" + const wantVersion = "NVRM version: NVIDIA UNIX x86_64 Kernel Module 610.57.04\n" + + for _, tc := range []struct { + name string + setup func(t *testing.T, dir string) + wantVersion string + }{ + { + name: "with_version", + setup: func(t *testing.T, dir string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, "version"), []byte(wantVersion), 0644); err != nil { + t.Fatalf("os.WriteFile(version) failed: %v", err) + } + }, + wantVersion: wantVersion, + }, + { + name: "without_version", + setup: func(t *testing.T, dir string) {}, + wantVersion: "", + }, + { + name: "unreadable_version", + setup: func(t *testing.T, dir string) { + t.Helper() + if err := os.Mkdir(filepath.Join(dir, "version"), 0755); err != nil { + t.Fatalf("os.Mkdir(version) failed: %v", err) + } + }, + wantVersion: "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "params"), []byte(wantParams), 0644); err != nil { + t.Fatalf("os.WriteFile(params) failed: %v", err) + } + tc.setup(t, dir) + + got, err := getHostSettingsFrom(dir, HostSettingsOptions{}) + if err != nil { + t.Fatalf("getHostSettingsFrom() failed: %v", err) + } + if got.ProcDriverNvidiaParams != wantParams { + t.Errorf("ProcDriverNvidiaParams = %q, want %q", got.ProcDriverNvidiaParams, wantParams) + } + if got.ProcDriverNvidiaVersion != tc.wantVersion { + t.Errorf("ProcDriverNvidiaVersion = %q, want %q", got.ProcDriverNvidiaVersion, tc.wantVersion) + } + }) + } +} + +func TestReadFileIfExists(t *testing.T) { + dir := t.TempDir() + existing := filepath.Join(dir, "existing") + const want = "hello\n" + if err := os.WriteFile(existing, []byte(want), 0644); err != nil { + t.Fatalf("os.WriteFile() failed: %v", err) + } + + got, err := readFileIfExists(existing) + if err != nil { + t.Fatalf("readFileIfExists(%q) failed: %v", existing, err) + } + if got != want { + t.Errorf("readFileIfExists(%q) = %q, want %q", existing, got, want) + } + + missing := filepath.Join(dir, "missing") + got, err = readFileIfExists(missing) + if err != nil { + t.Fatalf("readFileIfExists(%q) failed: %v", missing, err) + } + if got != "" { + t.Errorf("readFileIfExists(%q) = %q, want empty", missing, got) + } + + subdir := filepath.Join(dir, "subdir") + if err := os.Mkdir(subdir, 0755); err != nil { + t.Fatalf("os.Mkdir() failed: %v", err) + } + got, err = readFileIfExists(subdir) + if err == nil { + t.Errorf("readFileIfExists(%q) succeeded with %q, want error", subdir, got) + } +} diff --git a/pkg/sentry/devices/nvproxy/nvproxy.go b/pkg/sentry/devices/nvproxy/nvproxy.go index 4990e36cd3d..a11e95f631a 100644 --- a/pkg/sentry/devices/nvproxy/nvproxy.go +++ b/pkg/sentry/devices/nvproxy/nvproxy.go @@ -81,13 +81,14 @@ func Register(vfsObj *vfs.VirtualFilesystem, opts *Options) (*DeviceInfo, error) log.Warningf("nvproxy: NVIDIA driver capability set is empty; all GPU operations will fail") } nvp := &nvproxy{ - abi: abiEntry.cons(), - version: opts.DriverVersion, - capsEnabled: opts.DriverCaps, - useDevGofer: opts.UseDevGofer, - procDriverNvidiaParams: opts.HostSettings.ProcDriverNvidiaParams, - frontendFDs: make(map[*frontendFD]struct{}), - clients: make(map[nvgpu.Handle]*rootClient), + abi: abiEntry.cons(), + version: opts.DriverVersion, + capsEnabled: opts.DriverCaps, + useDevGofer: opts.UseDevGofer, + procDriverNvidiaParams: opts.HostSettings.ProcDriverNvidiaParams, + procDriverNvidiaVersion: opts.HostSettings.ProcDriverNvidiaVersion, + frontendFDs: make(map[*frontendFD]struct{}), + clients: make(map[nvgpu.Handle]*rootClient), } // Force ModifyDeviceFiles in /proc/driver/nvidia/params to 0. This is // consistent with libnvidia-container's src/nvc_mount.c:mount_procfs(). @@ -237,13 +238,14 @@ func DeviceInfoFromVFS(vfsObj *vfs.VirtualFilesystem) *DeviceInfo { // +stateify savable type nvproxy struct { - abi *driverABI `state:"nosave"` - version nvconf.DriverVersion - capsEnabled nvconf.DriverCaps - useDevGofer bool - procDriverNvidiaParams string - devInfo DeviceInfo - regularDevs [nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX + 1]*frontendDevice + abi *driverABI `state:"nosave"` + version nvconf.DriverVersion + capsEnabled nvconf.DriverCaps + useDevGofer bool + procDriverNvidiaParams string + procDriverNvidiaVersion string + devInfo DeviceInfo + regularDevs [nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX + 1]*frontendDevice fdsMu fdsMutex `state:"nosave"` frontendFDs map[*frontendFD]struct{} diff --git a/pkg/sentry/devices/nvproxy/procfs.go b/pkg/sentry/devices/nvproxy/procfs.go index 12dc03c332e..059ecda0c0f 100644 --- a/pkg/sentry/devices/nvproxy/procfs.go +++ b/pkg/sentry/devices/nvproxy/procfs.go @@ -40,6 +40,12 @@ func ProcfsInfoFromVFS(vfsObj *vfs.VirtualFilesystem) *ProcfsInfo { "params": nvp.procDriverNvidiaParams, }, } + // The version file exists only if the host has one, and holds the host's + // contents unchanged, as mount_procfs() in libnvidia-container's + // src/nvc_mount.c copies it. + if nvp.procDriverNvidiaVersion != "" { + procfsInfo.StaticFiles["version"] = nvp.procDriverNvidiaVersion + } if nvp.devInfo.HaveFabricIMEXManagement { procfsInfo.StaticFiles["capabilities/fabric-imex-mgmt"] = procfsCapability(nvp.devInfo.FabricIMEXManagementDevMinor, 0o400) } diff --git a/pkg/sentry/devices/nvproxy/procfs_test.go b/pkg/sentry/devices/nvproxy/procfs_test.go new file mode 100644 index 00000000000..1e1cfe41620 --- /dev/null +++ b/pkg/sentry/devices/nvproxy/procfs_test.go @@ -0,0 +1,77 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nvproxy + +import ( + "testing" + + "gvisor.dev/gvisor/pkg/sentry/contexttest" + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/pkg/sentry/vfs" +) + +// TestProcfsVersionIsHostVersion checks that /proc/driver/nvidia/version +// holds the host's file verbatim, and does not exist when the host has none. +func TestProcfsVersionIsHostVersion(t *testing.T) { + const hostVersion = "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 610.57.04 Release Build (dvs-builder@U22-I3-AF03-09-1) Thu Oct 2 17:07:40 UTC 2026\nGCC version: gcc version 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04)\n" + Init() + var driverVersion nvconf.DriverVersion + for version := range abis { + driverVersion = version + break + } + for _, tc := range []struct { + name string + hostVersion string + wantExists bool + }{ + { + name: "host_has_version", + hostVersion: hostVersion, + wantExists: true, + }, + { + name: "host_has_no_version", + hostVersion: "", + wantExists: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := contexttest.Context(t) + vfsObj := &vfs.VirtualFilesystem{} + if err := vfsObj.Init(ctx); err != nil { + t.Fatalf("vfsObj.Init() failed: %v", err) + } + if _, err := Register(vfsObj, &Options{ + DriverVersion: driverVersion, + AllowUnsupportedDriver: true, + DriverCaps: nvconf.CapCompute | nvconf.CapUtility, + HostSettings: &nvconf.HostSettings{ + ProcDriverNvidiaParams: "ModifyDeviceFiles: 1\n", + ProcDriverNvidiaVersion: tc.hostVersion, + }, + }); err != nil { + t.Fatalf("Register() failed: %v", err) + } + got, exists := ProcfsInfoFromVFS(vfsObj).StaticFiles["version"] + if exists != tc.wantExists { + t.Fatalf("version exists = %t, want %t", exists, tc.wantExists) + } + if got != tc.hostVersion { + t.Errorf("version = %q, want %q", got, tc.hostVersion) + } + }) + } +} diff --git a/runsc/cmd/sentry/sentrycmd/BUILD b/runsc/cmd/sentry/sentrycmd/BUILD index 7bb749aaf7a..cc77bbba5a8 100644 --- a/runsc/cmd/sentry/sentrycmd/BUILD +++ b/runsc/cmd/sentry/sentrycmd/BUILD @@ -56,10 +56,15 @@ go_library( go_test( name = "sentrycmd_test", size = "small", - srcs = ["chroot_test.go"], + srcs = [ + "boot_test.go", + "chroot_test.go", + ], embed = [":sentrycmd"], deps = [ + "//pkg/sentry/devices/nvproxy/nvconf", "//runsc/config", + "//runsc/flag", "//runsc/specutils", "@com_github_opencontainers_runtime_spec//specs-go:go_default_library", "@org_golang_x_sys//unix:go_default_library", diff --git a/runsc/cmd/sentry/sentrycmd/boot.go b/runsc/cmd/sentry/sentrycmd/boot.go index a0b1511afa2..1600688017f 100644 --- a/runsc/cmd/sentry/sentrycmd/boot.go +++ b/runsc/cmd/sentry/sentrycmd/boot.go @@ -235,6 +235,7 @@ type Boot struct { // here because nvconf.HostSettings.FabricIMEXManagementDevMinor in // particular is uint32 and there is no flag.Uint32Var(). procDriverNvidiaParams string + procDriverNvidiaVersion string nvidiaFabricIMEXManagementDevMinor int64 nvidiaMNCCManagementDevMinor int64 @@ -328,10 +329,37 @@ func (b *Boot) SetFlags(f *flag.FlagSet) { // Nvidia driver properties. f.StringVar(&b.nvidiaDriverVersion, "nvidia-driver-version", "", "Nvidia driver ABI version to use, or 'latest' for the newest supported one") f.StringVar(&b.procDriverNvidiaParams, "nvidia-host-params", "", "value of /proc/driver/nvidia/params on the host") + f.StringVar(&b.procDriverNvidiaVersion, "nvidia-host-version", "", "value of /proc/driver/nvidia/version on the host, or empty if the host has none") f.Int64Var(&b.nvidiaFabricIMEXManagementDevMinor, "nvidia-fabric-imex-mgmt-minor", -1, "DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt on the host") f.Int64Var(&b.nvidiaMNCCManagementDevMinor, "nvidia-mncc-mgmt-minor", -1, "DeviceFileMinor in /proc/driver/nvidia/capabilities/mncc-mgmt on the host") } +func (b *Boot) applyNvidiaHostSettings(nvhs *nvconf.HostSettings, argOverride map[string]string) { + b.procDriverNvidiaParams = nvhs.ProcDriverNvidiaParams + argOverride["nvidia-host-params"] = nvhs.ProcDriverNvidiaParams + b.procDriverNvidiaVersion = nvhs.ProcDriverNvidiaVersion + argOverride["nvidia-host-version"] = nvhs.ProcDriverNvidiaVersion + if nvhs.HaveFabricIMEXManagement { + b.nvidiaFabricIMEXManagementDevMinor = int64(nvhs.FabricIMEXManagementDevMinor) + argOverride["nvidia-fabric-imex-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.FabricIMEXManagementDevMinor), 10) + } + if nvhs.HaveMNCCManagement { + b.nvidiaMNCCManagementDevMinor = int64(nvhs.MNCCManagementDevMinor) + argOverride["nvidia-mncc-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.MNCCManagementDevMinor), 10) + } +} + +func (b *Boot) nvidiaHostSettings() *nvconf.HostSettings { + return &nvconf.HostSettings{ + ProcDriverNvidiaParams: b.procDriverNvidiaParams, + ProcDriverNvidiaVersion: b.procDriverNvidiaVersion, + HaveFabricIMEXManagement: b.nvidiaFabricIMEXManagementDevMinor >= 0, + FabricIMEXManagementDevMinor: uint32(b.nvidiaFabricIMEXManagementDevMinor), + HaveMNCCManagement: b.nvidiaMNCCManagementDevMinor >= 0, + MNCCManagementDevMinor: uint32(b.nvidiaMNCCManagementDevMinor), + } +} + // willReexec returns true if this boot process will re-execute itself to // apply credential/capability changes to all of its threads, rather than // applying capabilities to all threads in-process. @@ -462,16 +490,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma if err != nil { log.Warningf("Failed to get nvconf.HostSettings: %v", err) } else { - b.procDriverNvidiaParams = nvhs.ProcDriverNvidiaParams - argOverride["nvidia-host-params"] = nvhs.ProcDriverNvidiaParams - if nvhs.HaveFabricIMEXManagement { - b.nvidiaFabricIMEXManagementDevMinor = int64(nvhs.FabricIMEXManagementDevMinor) - argOverride["nvidia-fabric-imex-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.FabricIMEXManagementDevMinor), 10) - } - if nvhs.HaveMNCCManagement { - b.nvidiaMNCCManagementDevMinor = int64(nvhs.MNCCManagementDevMinor) - argOverride["nvidia-mncc-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.MNCCManagementDevMinor), 10) - } + b.applyNvidiaHostSettings(nvhs, argOverride) } } timer.Reached("set nvproxy flags") @@ -718,41 +737,35 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma // Create the loader. bootArgs := boot.Args{ - ID: f.Arg(0), - Spec: spec, - Conf: conf, - ControllerFD: b.controllerFD, - Device: fd.New(b.deviceFD), - PinRingFD: b.pinRingFD, - CPUDMALatencyFD: b.cpuDMALatencyFD, - GoferFDs: b.ioFDs.GetArray(), - DevGoferFD: b.devIoFD, - StdioFDs: b.stdioFDs.GetArray(), - PassFDs: b.passFDs.GetArray(), - ExecFD: b.execFD, - ProcSelfFD: procSelfFD, - GoferFilestoreFDs: b.goferFilestoreFDs.GetArray(), - GoferMountConfs: b.goferMountConfs.GetArray(), - NumCPU: b.cpuNum, - LifecycleNumCPU: b.lifecycleCPUNum, - CPUQuota: b.cpuQuota, - CPUPeriod: b.cpuPeriod, - TotalMem: b.totalMem, - TotalHostMem: b.totalHostMem, - UserLogFD: b.userLogFD, - RDMASysfs: rdmaSnap, - ProductName: b.productName, - PodInitConfigFD: b.podInitConfigFD, - SinkFDs: b.sinkFDs.GetArray(), - ProfileOpts: profile.MakeOpts(&b.profileFDs, conf.ProfileGCInterval), - NvidiaDriverVersion: nvidiaDriverVersion, - NvidiaHostSettings: &nvconf.HostSettings{ - ProcDriverNvidiaParams: b.procDriverNvidiaParams, - HaveFabricIMEXManagement: b.nvidiaFabricIMEXManagementDevMinor >= 0, - FabricIMEXManagementDevMinor: uint32(b.nvidiaFabricIMEXManagementDevMinor), - HaveMNCCManagement: b.nvidiaMNCCManagementDevMinor >= 0, - MNCCManagementDevMinor: uint32(b.nvidiaMNCCManagementDevMinor), - }, + ID: f.Arg(0), + Spec: spec, + Conf: conf, + ControllerFD: b.controllerFD, + Device: fd.New(b.deviceFD), + PinRingFD: b.pinRingFD, + CPUDMALatencyFD: b.cpuDMALatencyFD, + GoferFDs: b.ioFDs.GetArray(), + DevGoferFD: b.devIoFD, + StdioFDs: b.stdioFDs.GetArray(), + PassFDs: b.passFDs.GetArray(), + ExecFD: b.execFD, + ProcSelfFD: procSelfFD, + GoferFilestoreFDs: b.goferFilestoreFDs.GetArray(), + GoferMountConfs: b.goferMountConfs.GetArray(), + NumCPU: b.cpuNum, + LifecycleNumCPU: b.lifecycleCPUNum, + CPUQuota: b.cpuQuota, + CPUPeriod: b.cpuPeriod, + TotalMem: b.totalMem, + TotalHostMem: b.totalHostMem, + UserLogFD: b.userLogFD, + RDMASysfs: rdmaSnap, + ProductName: b.productName, + PodInitConfigFD: b.podInitConfigFD, + SinkFDs: b.sinkFDs.GetArray(), + ProfileOpts: profile.MakeOpts(&b.profileFDs, conf.ProfileGCInterval), + NvidiaDriverVersion: nvidiaDriverVersion, + NvidiaHostSettings: b.nvidiaHostSettings(), HostTHP: b.hostTHP, SaveFDs: b.saveFDs.GetFDs(), SaveCheckpointGofer: b.saveCheckpointGofer, diff --git a/runsc/cmd/sentry/sentrycmd/boot_test.go b/runsc/cmd/sentry/sentrycmd/boot_test.go new file mode 100644 index 00000000000..a485ba8f567 --- /dev/null +++ b/runsc/cmd/sentry/sentrycmd/boot_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sentrycmd + +import ( + "testing" + + "gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf" + "gvisor.dev/gvisor/runsc/flag" +) + +func TestBootNvidiaHostVersionFlag(t *testing.T) { + const wantVersion = "NVRM version: NVIDIA UNIX x86_64 Kernel Module 610.57.04\n" + b := &Boot{} + f := flag.NewFlagSet("test", flag.ContinueOnError) + b.SetFlags(f) + if err := f.Parse([]string{"--nvidia-host-version=" + wantVersion}); err != nil { + t.Fatalf("f.Parse() failed: %v", err) + } + if b.procDriverNvidiaVersion != wantVersion { + t.Errorf("procDriverNvidiaVersion = %q, want %q", b.procDriverNvidiaVersion, wantVersion) + } + if got := b.nvidiaHostSettings().ProcDriverNvidiaVersion; got != wantVersion { + t.Errorf("nvidiaHostSettings().ProcDriverNvidiaVersion = %q, want %q", got, wantVersion) + } +} + +func TestBootApplyNvidiaHostSettings(t *testing.T) { + b := &Boot{ + nvidiaFabricIMEXManagementDevMinor: -1, + nvidiaMNCCManagementDevMinor: -1, + } + argOverride := make(map[string]string) + want := &nvconf.HostSettings{ + ProcDriverNvidiaParams: "ModifyDeviceFiles: 1\n", + ProcDriverNvidiaVersion: "NVRM version: 610.57.04\n", + HaveFabricIMEXManagement: true, + FabricIMEXManagementDevMinor: 3, + HaveMNCCManagement: true, + MNCCManagementDevMinor: 4, + } + b.applyNvidiaHostSettings(want, argOverride) + + if argOverride["nvidia-host-params"] != want.ProcDriverNvidiaParams { + t.Errorf("argOverride[nvidia-host-params] = %q, want %q", argOverride["nvidia-host-params"], want.ProcDriverNvidiaParams) + } + if argOverride["nvidia-host-version"] != want.ProcDriverNvidiaVersion { + t.Errorf("argOverride[nvidia-host-version] = %q, want %q", argOverride["nvidia-host-version"], want.ProcDriverNvidiaVersion) + } + if argOverride["nvidia-fabric-imex-mgmt-minor"] != "3" { + t.Errorf("argOverride[nvidia-fabric-imex-mgmt-minor] = %q, want %q", argOverride["nvidia-fabric-imex-mgmt-minor"], "3") + } + if argOverride["nvidia-mncc-mgmt-minor"] != "4" { + t.Errorf("argOverride[nvidia-mncc-mgmt-minor] = %q, want %q", argOverride["nvidia-mncc-mgmt-minor"], "4") + } + + got := b.nvidiaHostSettings() + if *got != *want { + t.Errorf("nvidiaHostSettings() = %+v, want %+v", *got, *want) + } +} diff --git a/test/gpu/BUILD b/test/gpu/BUILD index cffaad91e43..0c6a3de37da 100644 --- a/test/gpu/BUILD +++ b/test/gpu/BUILD @@ -19,6 +19,20 @@ go_test( deps = ["//pkg/test/dockerutil"], ) +go_test( + name = "procfs_test", + srcs = ["procfs_test.go"], + # Needed to invalidate the bazel cache in case of any code changes. + data = ["//:release"], + tags = [ + "manual", + "noguitar", + "notap", + ], + visibility = ["//:sandbox"], + deps = ["//pkg/test/dockerutil"], +) + go_test( name = "pytorch_test", srcs = ["pytorch_test.go"], diff --git a/test/gpu/procfs_test.go b/test/gpu/procfs_test.go new file mode 100644 index 00000000000..9821c8a1e6f --- /dev/null +++ b/test/gpu/procfs_test.go @@ -0,0 +1,52 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package procfs_test tests the NVIDIA driver files in /proc/driver/nvidia. +package procfs_test + +import ( + "context" + "os" + "testing" + + "gvisor.dev/gvisor/pkg/test/dockerutil" +) + +// TestProcDriverNvidiaVersion checks that /proc/driver/nvidia/version in the +// container is the host's file verbatim. +func TestProcDriverNvidiaVersion(t *testing.T) { + want, err := os.ReadFile("/proc/driver/nvidia/version") + if err != nil { + t.Fatalf("failed to read the host's /proc/driver/nvidia/version: %v", err) + } + ctx := context.Background() + c := dockerutil.MakeContainer(ctx, t) + defer c.CleanUp(ctx) + + opts, err := dockerutil.GPURunOpts(dockerutil.SniffGPUOpts{ + Capabilities: "all", + DisableSnifferReason: "reads a procfs file and makes no ioctls", + }) + if err != nil { + t.Fatalf("failed to get GPU run options: %v", err) + } + opts.Image = "gpu/cuda-tests" + got, err := c.Run(ctx, opts, "cat", "/proc/driver/nvidia/version") + if err != nil { + t.Fatalf("could not read /proc/driver/nvidia/version: %v", err) + } + if got != string(want) { + t.Errorf("/proc/driver/nvidia/version = %q, want %q", got, string(want)) + } +}