diff --git a/docs/content/reference/gpu/nvidia.md b/docs/content/reference/gpu/nvidia.md index 2971d2ae88..a6c9621245 100644 --- a/docs/content/reference/gpu/nvidia.md +++ b/docs/content/reference/gpu/nvidia.md @@ -54,11 +54,10 @@ full sequence is: `/etc/cdi/nvidia.yaml`. Most CDI hooks are disabled to avoid interference with the nspawn environment. 5. **containerd runtime configuration.** Writes a containerd drop-in that - enables CDI support and registers the `nvidia` runtime class. On Azure - Linux hosts, the agent also selects the toolkit's CDI runtime mode so the - device plugin's default UUID environment variable resolves against the - generated driver-root CDI specification. Ubuntu retains the toolkit's - automatic runtime mode. + enables CDI support and registers the `nvidia` runtime class. After + successfully generating the host-derived CDI specification, the agent + selects the toolkit's CDI runtime mode so the device plugin's default + environment-variable request resolves against that specification. 6. **kubelet GPU advertisement.** Once kubelet starts and a user-deployed NVIDIA device plugin is running, GPUs are registered as `nvidia.com/gpu` extended resources on the node. @@ -90,6 +89,14 @@ copied libraries and their SONAME aliases. When the host provides version-compatible `nvidia-smi` to GPU containers. The binary is useful for diagnostics and monitoring but is not required for compute-only CUDA access. +Toolkit `auto` mode can interpret the device plugin's environment-variable +request as legacy injection. That bypasses the generated driver-root CDI edits, +including the `nvidia-smi` mount. Every NVIDIA nspawn node uses the prepared +driver root as its injection contract, so the agent selects CDI runtime mode +after generating the specification. If either generation or runtime +configuration fails, node startup stops rather than continuing with a mismatched +injection path. + ## Architecture Support Library discovery is architecture-aware. The agent uses `runtime.GOARCH` to diff --git a/pkg/agent/goalstates/nvidia.go b/pkg/agent/goalstates/nvidia.go index fa250c3339..9328a87b6c 100644 --- a/pkg/agent/goalstates/nvidia.go +++ b/pkg/agent/goalstates/nvidia.go @@ -51,10 +51,6 @@ type NvidiaHost struct { // was provisioned. Discovery refreshes must preserve this value. Required bool - // UseCDIRuntime selects CDI injection for Azure Linux NVIDIA workloads. - // Ubuntu retains the toolkit's existing automatic runtime mode. - UseCDIRuntime bool - // GPUDevicePaths lists NVIDIA GPU device paths discovered on the host // (e.g. /dev/nvidia0, /dev/nvidiactl, /dev/nvidia-caps, // /dev/nvidia-caps-imex-channels, /dev/dri/*). @@ -141,18 +137,12 @@ func ResolveNvidiaHost(arch string) (NvidiaHost, error) { } func resolveNvidiaHostWithReadiness(arch string, requireReadiness bool) (NvidiaHost, error) { - hostDistro, useCDIRuntime := detectNVIDIAHostOS() - initDeps := defaultNvidiaDeviceInitDeps() - initDeps.hostDistro = func() string { return hostDistro } - - return resolveNvidiaHost(arch, initDeps, nvidiaHostOptions{ - useCDIRuntime: useCDIRuntime, + return resolveNvidiaHost(arch, defaultNvidiaDeviceInitDeps(), nvidiaHostOptions{ requireReadiness: requireReadiness, }) } type nvidiaHostOptions struct { - useCDIRuntime bool requireReadiness bool } @@ -180,7 +170,6 @@ func resolveNvidiaHost(arch string, initDeps nvidiaDeviceInitDeps, opts nvidiaHo libraries := resolveNVIDIALibraries(archInfo) return NvidiaHost{ - UseCDIRuntime: opts.useCDIRuntime, GPUDevicePaths: devices, ContainerLibDir: archInfo.libDir, LibMappings: libraries.libMappings, @@ -194,28 +183,6 @@ func resolveNvidiaHost(arch string, initDeps nvidiaDeviceInitDeps, opts nvidiaHo }, nil } -func detectNVIDIAHostOS() (string, bool) { - data, err := os.ReadFile(hostOSReleasePath) - if err != nil { - return "", false - } - - values := osReleaseValues(data) - - return hostDistroFromOSReleaseValues(values), useNVIDIACDIRuntime(values) -} - -func useNVIDIACDIRuntime(osRelease map[string]string) bool { - if isAzureContainerLinux(osRelease) { - return false - } - - id := normalizeOSReleaseID(osRelease["ID"]) - - return (id == "azurelinux" || id == "azlinux") && - strings.HasPrefix(osRelease["VERSION_ID"], "3") -} - const ( nvidiaPCIVendorID = "0x10de" pciDisplayClassPrefix = "0x03" diff --git a/pkg/agent/goalstates/nvidia_test.go b/pkg/agent/goalstates/nvidia_test.go index 5ce7db63e8..29cf01e46e 100644 --- a/pkg/agent/goalstates/nvidia_test.go +++ b/pkg/agent/goalstates/nvidia_test.go @@ -85,12 +85,8 @@ func TestResolveNvidiaHostRefreshesDevicesAfterReadiness(t *testing.T) { return []byte("GPU 0: Tesla T4 (UUID: GPU-12345678-1234-1234-1234-123456789abc)\n"), nil } - host, err := resolveNvidiaHost("amd64", deps, nvidiaHostOptions{ - useCDIRuntime: true, - requireReadiness: true, - }) + host, err := resolveNvidiaHost("amd64", deps, nvidiaHostOptions{requireReadiness: true}) require.NoError(t, err) - require.True(t, host.UseCDIRuntime) require.Contains(t, host.GPUDevicePaths, filepath.Join(deviceDir, "nvidia0")) } @@ -118,64 +114,9 @@ func TestResolveNvidiaHostForPreflightDefersReadiness(t *testing.T) { host, err := resolveNvidiaHost("amd64", deps, nvidiaHostOptions{}) require.NoError(t, err) - require.False(t, host.UseCDIRuntime) require.Contains(t, host.GPUDevicePaths, filepath.Join(deviceDir, "nvidia0")) } -func TestAzureLinuxUsesNVIDIACDIRuntime(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - osRelease string - want bool - }{ - { - name: "Azure Linux 3", - osRelease: `ID=azurelinux -VERSION_ID="3.0"`, - want: true, - }, - { - name: "Azure Linux 3 alternate ID", - osRelease: `ID=azlinux -VERSION_ID="3.0"`, - want: true, - }, - { - name: "Azure Container Linux", - osRelease: `ID=azurelinux -VARIANT_ID=azurecontainerlinux -VERSION_ID="3.0.20260809"`, - }, - { - name: "Ubuntu", - osRelease: `ID=ubuntu -VERSION_ID="24.04"`, - }, - { - name: "Fedora compatibility fallback", - osRelease: `ID=fedora -VERSION_ID="44" -ID_LIKE="rhel fedora"`, - }, - { - name: "RHEL-like compatibility fallback", - osRelease: `ID=my-enterprise-linux -VERSION_ID="9" -ID_LIKE="rhel fedora"`, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - require.Equal(t, tt.want, useNVIDIACDIRuntime(osReleaseValues([]byte(tt.osRelease)))) - }) - } -} - func TestDiscoverNVIDIADevicesBindsCapabilityDirectories(t *testing.T) { t.Parallel() diff --git a/pkg/agent/phases/nodestart/nvidia.go b/pkg/agent/phases/nodestart/nvidia.go index 7c21d143da..1b01bdbf62 100644 --- a/pkg/agent/phases/nodestart/nvidia.go +++ b/pkg/agent/phases/nodestart/nvidia.go @@ -38,8 +38,8 @@ type setupNVIDIA struct { // with the nspawn environment (e.g. mounting tmpfs over /proc paths, CUDA // compat hooks on NVSwitch systems). Only create-symlinks and update-ldcache // are retained. -// 5. On Azure Linux, selects CDI runtime mode so UUID environment-variable -// requests from the device plugin resolve against the generated CDI spec. +// 5. Selects CDI runtime mode so environment-variable requests from the +// device plugin resolve against the generated CDI spec. // // This task must run after StartNSpawnMachine (the machine must be booted and // the /run/host-nvidia/ bind mounts must be active) and before StartKubelet @@ -84,10 +84,6 @@ func (s *setupNVIDIA) Do(ctx context.Context) error { } func (s *setupNVIDIA) configureContainerRuntime(ctx context.Context) error { - if !s.goalState.Nvidia.UseCDIRuntime { - return nil - } - if _, err := s.run( ctx, s.log, s.goalState.MachineName, goalstates.NvidiaCTKPath, "config", diff --git a/pkg/agent/phases/nodestart/nvidia_test.go b/pkg/agent/phases/nodestart/nvidia_test.go index 565f307850..c8f220f63f 100644 --- a/pkg/agent/phases/nodestart/nvidia_test.go +++ b/pkg/agent/phases/nodestart/nvidia_test.go @@ -42,19 +42,14 @@ func TestConfigureNVIDIAContainerRuntime(t *testing.T) { tests := []struct { name string - enabled bool runErr error wantErr string - wantRun bool }{ - {name: "Azure Linux uses CDI", enabled: true, wantRun: true}, - {name: "Ubuntu remains automatic", enabled: false}, + {name: "successful setup uses CDI"}, { name: "configuration failure is explicit", - enabled: true, runErr: errors.New("exit status 1"), wantErr: "configure NVIDIA container runtime CDI mode", - wantRun: true, }, } @@ -68,9 +63,6 @@ func TestConfigureNVIDIAContainerRuntime(t *testing.T) { log: slog.New(slog.DiscardHandler), goalState: &goalstates.NodeStart{ MachineName: "kube1", - Nvidia: goalstates.NvidiaHost{ - UseCDIRuntime: tt.enabled, - }, }, run: func(_ context.Context, _ *slog.Logger, machine string, args ...string) (string, error) { require.Equal(t, "kube1", machine) @@ -88,17 +80,13 @@ func TestConfigureNVIDIAContainerRuntime(t *testing.T) { require.NoError(t, err) } - if tt.wantRun { - require.Equal(t, []string{ - goalstates.NvidiaCTKPath, - "config", - "--set", - "nvidia-container-runtime.mode=cdi", - "--in-place", - }, gotArgs) - } else { - require.Empty(t, gotArgs) - } + require.Equal(t, []string{ + goalstates.NvidiaCTKPath, + "config", + "--set", + "nvidia-container-runtime.mode=cdi", + "--in-place", + }, gotArgs) }) } }