Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,7 @@ l4-gpu-tests: l4-gpu-images $(RUNTIME_BIN)

gpu-all-tests: gpu-images gpu-smoke-tests $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME),--nvproxy=true --nvproxy-allowed-driver-capabilities=all)
@$(call sudo,test/gpu:procfs_test,--runtime=$(RUNTIME) -test.v $(ARGS))
@$(call sudo,test/gpu:pytorch_test,--runtime=$(RUNTIME) -test.v $(ARGS))
@$(call sudo,test/gpu:textgen_test,--runtime=$(RUNTIME) -test.v $(ARGS))
@$(call sudo,test/gpu:imagegen_test,--runtime=$(RUNTIME) -test.v $(ARGS))
Expand All @@ -420,6 +421,7 @@ gpu-all-tests: gpu-images gpu-smoke-tests $(RUNTIME_BIN)

cos-gpu-all-tests: gpu-images cos-gpu-smoke-tests $(RUNTIME_BIN)
@$(call install_runtime,$(RUNTIME),--nvproxy=true)
@$(call sudo,test/gpu:procfs_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
@$(call sudo,test/gpu:pytorch_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
@$(call sudo,test/gpu:textgen_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
@$(call sudo,test/gpu:imagegen_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
Expand Down
7 changes: 6 additions & 1 deletion pkg/sentry/devices/nvproxy/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,17 @@ go_library(

go_test(
name = "nvproxy_test",
srcs = ["nvproxy_test.go"],
srcs = [
"nvproxy_test.go",
"procfs_test.go",
],
embed = [":nvproxy"],
deps = [
"//pkg/abi/nvgpu",
"//pkg/seccomp",
"//pkg/sentry/contexttest",
"//pkg/sentry/devices/nvproxy/nvconf",
"//pkg/sentry/vfs",
"@org_golang_x_sys//unix:go_default_library",
],
)
Expand Down
5 changes: 4 additions & 1 deletion pkg/sentry/devices/nvproxy/nvconf/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ go_library(
go_test(
name = "nvconf_test",
size = "small",
srcs = ["caps_test.go"],
srcs = [
"caps_test.go",
"hostsettings_test.go",
],
embed = [":nvconf"],
)
48 changes: 40 additions & 8 deletions pkg/sentry/devices/nvproxy/nvconf/hostsettings.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"

Expand All @@ -30,6 +31,10 @@ type HostSettings struct {
// ProcDriverNvidiaParams is the contents of /proc/driver/nvidia/params.
ProcDriverNvidiaParams string

// ProcDriverNvidiaVersion is the contents of /proc/driver/nvidia/version,
// or empty if the host has no such file or it cannot be read.
ProcDriverNvidiaVersion string

// If HaveFabricIMEXManagement is true, FabricIMEXManagementDevMinor is the
// device minor number advertised in
// /proc/driver/nvidia/capabilities/fabric-imex-mgmt.
Expand All @@ -52,23 +57,36 @@ type HostSettingsOptions struct {

// GetHostSettings returns HostSettings.
func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) {
return getHostSettingsFrom("/proc/driver/nvidia", opts)
}

func getHostSettingsFrom(nvidiaProcDir string, opts HostSettingsOptions) (*HostSettings, error) {
settings := &HostSettings{}

params, err := os.ReadFile("/proc/driver/nvidia/params")
paramsPath := filepath.Join(nvidiaProcDir, "params")
params, err := os.ReadFile(paramsPath)
if err != nil {
return nil, fmt.Errorf("failed to read /proc/driver/nvidia/params: %w", err)
return nil, fmt.Errorf("failed to read %s: %w", paramsPath, err)
}
settings.ProcDriverNvidiaParams = string(params)

versionPath := filepath.Join(nvidiaProcDir, "version")
version, err := readFileIfExists(versionPath)
if err != nil {
log.Warningf("Not serving %s: %v", versionPath, err)
}
settings.ProcDriverNvidiaVersion = version

if opts.WantFabricIMEXManagement {
fabricImexMgmt, err := os.ReadFile("/proc/driver/nvidia/capabilities/fabric-imex-mgmt")
fabricImexPath := filepath.Join(nvidiaProcDir, "capabilities", "fabric-imex-mgmt")
fabricImexMgmt, err := os.ReadFile(fabricImexPath)
if err != nil {
return nil, fmt.Errorf("failed to read /proc/driver/nvidia/capabilities/fabric-imex-mgmt: %w", err)
return nil, fmt.Errorf("failed to read %s: %w", fabricImexPath, err)
}
minorRe := regexp.MustCompile(`DeviceFileMinor: (\d+)`)
m := minorRe.FindSubmatch(fabricImexMgmt)
if m == nil {
return nil, fmt.Errorf("failed to find DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt")
return nil, fmt.Errorf("failed to find DeviceFileMinor in %s", fabricImexPath)
}
minor, err := strconv.ParseUint(string(m[1]), 10, 32)
if err != nil {
Expand All @@ -77,10 +95,11 @@ func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) {
settings.HaveFabricIMEXManagement = true
settings.FabricIMEXManagementDevMinor = uint32(minor)

if mnccMgmt, err := os.ReadFile("/proc/driver/nvidia/capabilities/mncc-mgmt"); err == nil {
mnccPath := filepath.Join(nvidiaProcDir, "capabilities", "mncc-mgmt")
if mnccMgmt, err := os.ReadFile(mnccPath); err == nil {
m := minorRe.FindSubmatch(mnccMgmt)
if m == nil {
return nil, fmt.Errorf("failed to find DeviceFileMinor in /proc/driver/nvidia/capabilities/mncc-mgmt")
return nil, fmt.Errorf("failed to find DeviceFileMinor in %s", mnccPath)
}
mnccMinor, err := strconv.ParseUint(string(m[1]), 10, 32)
if err != nil {
Expand All @@ -89,13 +108,26 @@ func GetHostSettings(opts HostSettingsOptions) (*HostSettings, error) {
settings.HaveMNCCManagement = true
settings.MNCCManagementDevMinor = uint32(mnccMinor)
} else if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("failed to read /proc/driver/nvidia/capabilities/mncc-mgmt: %w", err)
return nil, fmt.Errorf("failed to read %s: %w", mnccPath, err)
}
}

return settings, nil
}

// readFileIfExists returns the contents of the file at path, or an empty
// string if no file exists at path.
func readFileIfExists(path string) (string, error) {
data, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return "", nil
}
if err != nil {
return "", fmt.Errorf("failed to read %s: %w", path, err)
}
return string(data), nil
}

// IMEXChannelCount returns the number of IMEX channels indicated by
// /proc/driver/nvidia/params. See description of NVreg_ImexChannelCount in the
// Nvidia GPU driver's kernel-open/nvidia/nv-reg.h.
Expand Down
112 changes: 112 additions & 0 deletions pkg/sentry/devices/nvproxy/nvconf/hostsettings_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// Copyright 2026 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nvconf

import (
"os"
"path/filepath"
"testing"
)

func TestGetHostSettingsFrom(t *testing.T) {
const wantParams = "ModifyDeviceFiles: 1\nImexChannelCount: 4\n"
const wantVersion = "NVRM version: NVIDIA UNIX x86_64 Kernel Module 610.57.04\n"

for _, tc := range []struct {
name string
setup func(t *testing.T, dir string)
wantVersion string
}{
{
name: "with_version",
setup: func(t *testing.T, dir string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, "version"), []byte(wantVersion), 0644); err != nil {
t.Fatalf("os.WriteFile(version) failed: %v", err)
}
},
wantVersion: wantVersion,
},
{
name: "without_version",
setup: func(t *testing.T, dir string) {},
wantVersion: "",
},
{
name: "unreadable_version",
setup: func(t *testing.T, dir string) {
t.Helper()
if err := os.Mkdir(filepath.Join(dir, "version"), 0755); err != nil {
t.Fatalf("os.Mkdir(version) failed: %v", err)
}
},
wantVersion: "",
},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "params"), []byte(wantParams), 0644); err != nil {
t.Fatalf("os.WriteFile(params) failed: %v", err)
}
tc.setup(t, dir)

got, err := getHostSettingsFrom(dir, HostSettingsOptions{})
if err != nil {
t.Fatalf("getHostSettingsFrom() failed: %v", err)
}
if got.ProcDriverNvidiaParams != wantParams {
t.Errorf("ProcDriverNvidiaParams = %q, want %q", got.ProcDriverNvidiaParams, wantParams)
}
if got.ProcDriverNvidiaVersion != tc.wantVersion {
t.Errorf("ProcDriverNvidiaVersion = %q, want %q", got.ProcDriverNvidiaVersion, tc.wantVersion)
}
})
}
}

func TestReadFileIfExists(t *testing.T) {
dir := t.TempDir()
existing := filepath.Join(dir, "existing")
const want = "hello\n"
if err := os.WriteFile(existing, []byte(want), 0644); err != nil {
t.Fatalf("os.WriteFile() failed: %v", err)
}

got, err := readFileIfExists(existing)
if err != nil {
t.Fatalf("readFileIfExists(%q) failed: %v", existing, err)
}
if got != want {
t.Errorf("readFileIfExists(%q) = %q, want %q", existing, got, want)
}

missing := filepath.Join(dir, "missing")
got, err = readFileIfExists(missing)
if err != nil {
t.Fatalf("readFileIfExists(%q) failed: %v", missing, err)
}
if got != "" {
t.Errorf("readFileIfExists(%q) = %q, want empty", missing, got)
}

subdir := filepath.Join(dir, "subdir")
if err := os.Mkdir(subdir, 0755); err != nil {
t.Fatalf("os.Mkdir() failed: %v", err)
}
got, err = readFileIfExists(subdir)
if err == nil {
t.Errorf("readFileIfExists(%q) succeeded with %q, want error", subdir, got)
}
}
30 changes: 16 additions & 14 deletions pkg/sentry/devices/nvproxy/nvproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,14 @@ func Register(vfsObj *vfs.VirtualFilesystem, opts *Options) (*DeviceInfo, error)
log.Warningf("nvproxy: NVIDIA driver capability set is empty; all GPU operations will fail")
}
nvp := &nvproxy{
abi: abiEntry.cons(),
version: opts.DriverVersion,
capsEnabled: opts.DriverCaps,
useDevGofer: opts.UseDevGofer,
procDriverNvidiaParams: opts.HostSettings.ProcDriverNvidiaParams,
frontendFDs: make(map[*frontendFD]struct{}),
clients: make(map[nvgpu.Handle]*rootClient),
abi: abiEntry.cons(),
version: opts.DriverVersion,
capsEnabled: opts.DriverCaps,
useDevGofer: opts.UseDevGofer,
procDriverNvidiaParams: opts.HostSettings.ProcDriverNvidiaParams,
procDriverNvidiaVersion: opts.HostSettings.ProcDriverNvidiaVersion,
frontendFDs: make(map[*frontendFD]struct{}),
clients: make(map[nvgpu.Handle]*rootClient),
}
// Force ModifyDeviceFiles in /proc/driver/nvidia/params to 0. This is
// consistent with libnvidia-container's src/nvc_mount.c:mount_procfs().
Expand Down Expand Up @@ -237,13 +238,14 @@ func DeviceInfoFromVFS(vfsObj *vfs.VirtualFilesystem) *DeviceInfo {

// +stateify savable
type nvproxy struct {
abi *driverABI `state:"nosave"`
version nvconf.DriverVersion
capsEnabled nvconf.DriverCaps
useDevGofer bool
procDriverNvidiaParams string
devInfo DeviceInfo
regularDevs [nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX + 1]*frontendDevice
abi *driverABI `state:"nosave"`
version nvconf.DriverVersion
capsEnabled nvconf.DriverCaps
useDevGofer bool
procDriverNvidiaParams string
procDriverNvidiaVersion string
devInfo DeviceInfo
regularDevs [nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX + 1]*frontendDevice

fdsMu fdsMutex `state:"nosave"`
frontendFDs map[*frontendFD]struct{}
Expand Down
6 changes: 6 additions & 0 deletions pkg/sentry/devices/nvproxy/procfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ func ProcfsInfoFromVFS(vfsObj *vfs.VirtualFilesystem) *ProcfsInfo {
"params": nvp.procDriverNvidiaParams,
},
}
// The version file exists only if the host has one, and holds the host's
// contents unchanged, as mount_procfs() in libnvidia-container's
// src/nvc_mount.c copies it.
if nvp.procDriverNvidiaVersion != "" {
procfsInfo.StaticFiles["version"] = nvp.procDriverNvidiaVersion
}
if nvp.devInfo.HaveFabricIMEXManagement {
procfsInfo.StaticFiles["capabilities/fabric-imex-mgmt"] = procfsCapability(nvp.devInfo.FabricIMEXManagementDevMinor, 0o400)
}
Expand Down
77 changes: 77 additions & 0 deletions pkg/sentry/devices/nvproxy/procfs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Copyright 2026 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nvproxy

import (
"testing"

"gvisor.dev/gvisor/pkg/sentry/contexttest"
"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf"
"gvisor.dev/gvisor/pkg/sentry/vfs"
)

// TestProcfsVersionIsHostVersion checks that /proc/driver/nvidia/version
// holds the host's file verbatim, and does not exist when the host has none.
func TestProcfsVersionIsHostVersion(t *testing.T) {
const hostVersion = "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 610.57.04 Release Build (dvs-builder@U22-I3-AF03-09-1) Thu Oct 2 17:07:40 UTC 2026\nGCC version: gcc version 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04)\n"
Init()
var driverVersion nvconf.DriverVersion
for version := range abis {
driverVersion = version
break
}
for _, tc := range []struct {
name string
hostVersion string
wantExists bool
}{
{
name: "host_has_version",
hostVersion: hostVersion,
wantExists: true,
},
{
name: "host_has_no_version",
hostVersion: "",
wantExists: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := contexttest.Context(t)
vfsObj := &vfs.VirtualFilesystem{}
if err := vfsObj.Init(ctx); err != nil {
t.Fatalf("vfsObj.Init() failed: %v", err)
}
if _, err := Register(vfsObj, &Options{
DriverVersion: driverVersion,
AllowUnsupportedDriver: true,
DriverCaps: nvconf.CapCompute | nvconf.CapUtility,
HostSettings: &nvconf.HostSettings{
ProcDriverNvidiaParams: "ModifyDeviceFiles: 1\n",
ProcDriverNvidiaVersion: tc.hostVersion,
},
}); err != nil {
t.Fatalf("Register() failed: %v", err)
}
got, exists := ProcfsInfoFromVFS(vfsObj).StaticFiles["version"]
if exists != tc.wantExists {
t.Fatalf("version exists = %t, want %t", exists, tc.wantExists)
}
if got != tc.hostVersion {
t.Errorf("version = %q, want %q", got, tc.hostVersion)
}
})
}
}
Loading
Loading