fix(agent): use CDI runtime for NVIDIA nspawn nodes

docs/content/reference/gpu/nvidia.md

@@ -54,11 +54,10 @@ full sequence is:
    `/etc/cdi/nvidia.yaml`. Most CDI hooks are disabled to avoid interference
    with the nspawn environment.
 5. **containerd runtime configuration.** Writes a containerd drop-in that
-   enables CDI support and registers the `nvidia` runtime class. On Azure
-   Linux hosts, the agent also selects the toolkit's CDI runtime mode so the
-   device plugin's default UUID environment variable resolves against the
-   generated driver-root CDI specification. Ubuntu retains the toolkit's
-   automatic runtime mode.
+   enables CDI support and registers the `nvidia` runtime class. After
+   successfully generating the host-derived CDI specification, the agent
+   selects the toolkit's CDI runtime mode so the device plugin's default
+   environment-variable request resolves against that specification.
 6. **kubelet GPU advertisement.** Once kubelet starts and a user-deployed
    NVIDIA device plugin is running, GPUs are registered as `nvidia.com/gpu`
    extended resources on the node.
@@ -90,6 +89,14 @@ copied libraries and their SONAME aliases. When the host provides
 version-compatible `nvidia-smi` to GPU containers. The binary is useful for
 diagnostics and monitoring but is not required for compute-only CUDA access.
 
+Toolkit `auto` mode can interpret the device plugin's environment-variable
+request as legacy injection. That bypasses the generated driver-root CDI edits,
+including the `nvidia-smi` mount. Every NVIDIA nspawn node uses the prepared
+driver root as its injection contract, so the agent selects CDI runtime mode
+after generating the specification. If either generation or runtime
+configuration fails, node startup stops rather than continuing with a mismatched
+injection path.
+
 ## Architecture Support
 
 Library discovery is architecture-aware. The agent uses `runtime.GOARCH` to

pkg/agent/goalstates/nvidia.go

@@ -51,10 +51,6 @@ type NvidiaHost struct {
 	// was provisioned. Discovery refreshes must preserve this value.
 	Required bool
 
-	// UseCDIRuntime selects CDI injection for Azure Linux NVIDIA workloads.
-	// Ubuntu retains the toolkit's existing automatic runtime mode.
-	UseCDIRuntime bool
-
 	// GPUDevicePaths lists NVIDIA GPU device paths discovered on the host
 	// (e.g. /dev/nvidia0, /dev/nvidiactl, /dev/nvidia-caps,
 	// /dev/nvidia-caps-imex-channels, /dev/dri/*).
@@ -141,18 +137,12 @@ func ResolveNvidiaHost(arch string) (NvidiaHost, error) {
 }
 
 func resolveNvidiaHostWithReadiness(arch string, requireReadiness bool) (NvidiaHost, error) {
-	hostDistro, useCDIRuntime := detectNVIDIAHostOS()
-	initDeps := defaultNvidiaDeviceInitDeps()
-	initDeps.hostDistro = func() string { return hostDistro }
-
-	return resolveNvidiaHost(arch, initDeps, nvidiaHostOptions{
-		useCDIRuntime:    useCDIRuntime,
+	return resolveNvidiaHost(arch, defaultNvidiaDeviceInitDeps(), nvidiaHostOptions{
 		requireReadiness: requireReadiness,
 	})
 }
 
 type nvidiaHostOptions struct {
-	useCDIRuntime    bool
 	requireReadiness bool
 }
 
@@ -180,7 +170,6 @@ func resolveNvidiaHost(arch string, initDeps nvidiaDeviceInitDeps, opts nvidiaHo
 	libraries := resolveNVIDIALibraries(archInfo)
 
 	return NvidiaHost{
-		UseCDIRuntime:     opts.useCDIRuntime,
 		GPUDevicePaths:    devices,
 		ContainerLibDir:   archInfo.libDir,
 		LibMappings:       libraries.libMappings,
@@ -194,28 +183,6 @@ func resolveNvidiaHost(arch string, initDeps nvidiaDeviceInitDeps, opts nvidiaHo
 	}, nil
 }
 
-func detectNVIDIAHostOS() (string, bool) {
-	data, err := os.ReadFile(hostOSReleasePath)
-	if err != nil {
-		return "", false
-	}
-
-	values := osReleaseValues(data)
-
-	return hostDistroFromOSReleaseValues(values), useNVIDIACDIRuntime(values)
-}
-
-func useNVIDIACDIRuntime(osRelease map[string]string) bool {
-	if isAzureContainerLinux(osRelease) {
-		return false
-	}
-
-	id := normalizeOSReleaseID(osRelease["ID"])
-
-	return (id == "azurelinux" || id == "azlinux") &&
-		strings.HasPrefix(osRelease["VERSION_ID"], "3")
-}
-
 const (
 	nvidiaPCIVendorID       = "0x10de"
 	pciDisplayClassPrefix   = "0x03"

pkg/agent/goalstates/nvidia_test.go

@@ -85,12 +85,8 @@ func TestResolveNvidiaHostRefreshesDevicesAfterReadiness(t *testing.T) {
 		return []byte("GPU 0: Tesla T4 (UUID: GPU-12345678-1234-1234-1234-123456789abc)\n"), nil
 	}
 
-	host, err := resolveNvidiaHost("amd64", deps, nvidiaHostOptions{
-		useCDIRuntime:    true,
-		requireReadiness: true,
-	})
+	host, err := resolveNvidiaHost("amd64", deps, nvidiaHostOptions{requireReadiness: true})
 	require.NoError(t, err)
-	require.True(t, host.UseCDIRuntime)
 	require.Contains(t, host.GPUDevicePaths, filepath.Join(deviceDir, "nvidia0"))
 }
 
@@ -118,64 +114,9 @@ func TestResolveNvidiaHostForPreflightDefersReadiness(t *testing.T) {
 
 	host, err := resolveNvidiaHost("amd64", deps, nvidiaHostOptions{})
 	require.NoError(t, err)
-	require.False(t, host.UseCDIRuntime)
 	require.Contains(t, host.GPUDevicePaths, filepath.Join(deviceDir, "nvidia0"))
 }
 
-func TestAzureLinuxUsesNVIDIACDIRuntime(t *testing.T) {
-	t.Parallel()
-
-	tests := []struct {
-		name      string
-		osRelease string
-		want      bool
-	}{
-		{
-			name: "Azure Linux 3",
-			osRelease: `ID=azurelinux
-VERSION_ID="3.0"`,
-			want: true,
-		},
-		{
-			name: "Azure Linux 3 alternate ID",
-			osRelease: `ID=azlinux
-VERSION_ID="3.0"`,
-			want: true,
-		},
-		{
-			name: "Azure Container Linux",
-			osRelease: `ID=azurelinux
-VARIANT_ID=azurecontainerlinux
-VERSION_ID="3.0.20260809"`,
-		},
-		{
-			name: "Ubuntu",
-			osRelease: `ID=ubuntu
-VERSION_ID="24.04"`,
-		},
-		{
-			name: "Fedora compatibility fallback",
-			osRelease: `ID=fedora
-VERSION_ID="44"
-ID_LIKE="rhel fedora"`,
-		},
-		{
-			name: "RHEL-like compatibility fallback",
-			osRelease: `ID=my-enterprise-linux
-VERSION_ID="9"
-ID_LIKE="rhel fedora"`,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			require.Equal(t, tt.want, useNVIDIACDIRuntime(osReleaseValues([]byte(tt.osRelease))))
-		})
-	}
-}
-
 func TestDiscoverNVIDIADevicesBindsCapabilityDirectories(t *testing.T) {
 	t.Parallel()
 

pkg/agent/phases/nodestart/nvidia.go

@@ -38,8 +38,8 @@ type setupNVIDIA struct {
 //     with the nspawn environment (e.g. mounting tmpfs over /proc paths, CUDA
 //     compat hooks on NVSwitch systems). Only create-symlinks and update-ldcache
 //     are retained.
-//  5. On Azure Linux, selects CDI runtime mode so UUID environment-variable
-//     requests from the device plugin resolve against the generated CDI spec.
+//  5. Selects CDI runtime mode so environment-variable requests from the
+//     device plugin resolve against the generated CDI spec.
 //
 // This task must run after StartNSpawnMachine (the machine must be booted and
 // the /run/host-nvidia/ bind mounts must be active) and before StartKubelet
@@ -84,10 +84,6 @@ func (s *setupNVIDIA) Do(ctx context.Context) error {
 }
 
 func (s *setupNVIDIA) configureContainerRuntime(ctx context.Context) error {
-	if !s.goalState.Nvidia.UseCDIRuntime {
-		return nil
-	}
-
 	if _, err := s.run(
 		ctx, s.log, s.goalState.MachineName,
 		goalstates.NvidiaCTKPath, "config",

pkg/agent/phases/nodestart/nvidia_test.go

@@ -42,19 +42,14 @@ func TestConfigureNVIDIAContainerRuntime(t *testing.T) {
 
 	tests := []struct {
 		name    string
-		enabled bool
 		runErr  error
 		wantErr string
-		wantRun bool
 	}{
-		{name: "Azure Linux uses CDI", enabled: true, wantRun: true},
-		{name: "Ubuntu remains automatic", enabled: false},
+		{name: "successful setup uses CDI"},
 		{
 			name:    "configuration failure is explicit",
-			enabled: true,
 			runErr:  errors.New("exit status 1"),
 			wantErr: "configure NVIDIA container runtime CDI mode",
-			wantRun: true,
 		},
 	}
 
@@ -68,9 +63,6 @@ func TestConfigureNVIDIAContainerRuntime(t *testing.T) {
 				log: slog.New(slog.DiscardHandler),
 				goalState: &goalstates.NodeStart{
 					MachineName: "kube1",
-					Nvidia: goalstates.NvidiaHost{
-						UseCDIRuntime: tt.enabled,
-					},
 				},
 				run: func(_ context.Context, _ *slog.Logger, machine string, args ...string) (string, error) {
 					require.Equal(t, "kube1", machine)
@@ -88,17 +80,13 @@ func TestConfigureNVIDIAContainerRuntime(t *testing.T) {
 				require.NoError(t, err)
 			}
 
-			if tt.wantRun {
-				require.Equal(t, []string{
-					goalstates.NvidiaCTKPath,
-					"config",
-					"--set",
-					"nvidia-container-runtime.mode=cdi",
-					"--in-place",
-				}, gotArgs)
-			} else {
-				require.Empty(t, gotArgs)
-			}
+			require.Equal(t, []string{
+				goalstates.NvidiaCTKPath,
+				"config",
+				"--set",
+				"nvidia-container-runtime.mode=cdi",
+				"--in-place",
+			}, gotArgs)
 		})
 	}
 }