feat(provider): world-readable image fingerprint

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Unreleased
 
+- Write a world-readable `/run/nddev/image-fingerprint` from the root
+  assignment so an unprivileged job can attest the live image without the
+  Incus guest API, which answers HTTP 401 to uid runner. The parent
+  `/run/nddev` directory is created at assignment so current images do not
+  wait on a bake. This is `v0.1.5-nddev.123`.
+
 - Count only host-global kernel OOM kills (`constraint=CONSTRAINT_NONE`) into
   `gha_fleet_host_oom_kills_total`, and page `host_oom_detected` on the
   in-window span of that counter instead of `increase()`. `/proc/vmstat

config/example-runner-1.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.122
+  provider_version: v0.1.5-nddev.123
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-2.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.122
+  provider_version: v0.1.5-nddev.123
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-3.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.122
+  provider_version: v0.1.5-nddev.123
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-4.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.122
+  provider_version: v0.1.5-nddev.123
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-services.yaml

@@ -27,7 +27,7 @@ control_plane:
   manager_version: v0.2.1-nddev.88
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.122
+  provider_version: v0.1.5-nddev.123
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/provider-derivative.yaml

@@ -16,7 +16,7 @@ artifact: garm-provider-incus
 # state all move together, because all three derive from here. A provider change
 # that does not bump it ships under the previous version, which is exactly how
 # runner-1 and runner-2 diverged.
-derivative_version: v0.1.5-nddev.122
+derivative_version: v0.1.5-nddev.123
 
 # The external-provider protocol GARM speaks to this binary. It moves on its own
 # schedule -- a provider release does not imply an interface release -- so it is
@@ -37,8 +37,8 @@ runtime:
   queue_intent_schema_version: 6
 
 build:
-  source_commit: 40a8402d517fd8cdc1edb04ec27ec524a4546be1
-  binary_sha256: 8997d4f18550bdb94805cfe981390fb4ab2104460bb513f6c95017fac8b1a38b
+  source_commit: 7c6a1c68398cf07bd949daf265a9b319193034eb
+  binary_sha256: 1606c4fe06a326605fbd9d8cd85a5e3722213b9c4f1422afd59f1fa7d5a04058
   go_version: go1.26.7
   cgo_enabled: false
   target_os: linux

internal/garmproviderincus/provider/image_identity.go

@@ -0,0 +1,70 @@
+package provider
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"regexp"
+	"time"
+
+	incus "github.com/lxc/incus/v7/client"
+)
+
+const (
+	// Unprivileged GitHub jobs run as uid runner. The Incus guest API that
+	// exposes user.nddev.image-fingerprint answers HTTP 401 to every non-root
+	// caller, so a fail-closed guest-config read cannot attest the image.
+	// The root assignment writes this world-readable file instead. /run is
+	// tmpfs; the parent directory is created here so current images do not
+	// wait on a bake.
+	imageIdentityDirectory = "/run/nddev"
+	imageIdentityPath      = "/run/nddev/image-fingerprint"
+)
+
+var imageFingerprintPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+func (l *Incus) publishJobImageIdentity(ctx context.Context, cli InstanceServerInterface, instanceName, flavor string) error {
+	imagePolicy, err := l.workerImagePolicy(flavor)
+	if err != nil {
+		return err
+	}
+	return l.injectImageIdentity(ctx, cli, instanceName, imagePolicy.Fingerprint)
+}
+
+func (l *Incus) injectImageIdentity(ctx context.Context, cli InstanceServerInterface, instanceName, fingerprint string) error {
+	if !imageFingerprintPattern.MatchString(fingerprint) {
+		return fmt.Errorf("inject image identity: fingerprint %q is not a 64-hex digest", fingerprint)
+	}
+	body := []byte(fingerprint + "\n")
+	deadline := time.Now().Add(cacheInjectionTimeout)
+	for {
+		err := l.writeImageIdentity(cli, instanceName, body)
+		if err == nil {
+			return nil
+		}
+		instance, _, inspectErr := cli.GetInstanceFull(instanceName)
+		if inspectErr == nil && instance.State != nil && instance.State.Status != "Running" {
+			return fmt.Errorf("inject image identity: instance stopped: %w", err)
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("inject image identity: guest agent did not accept %s within %s: %w", imageIdentityPath, cacheInjectionTimeout, err)
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("inject image identity: %w", ctx.Err())
+		case <-time.After(time.Second):
+		}
+	}
+}
+
+func (l *Incus) writeImageIdentity(cli InstanceServerInterface, instanceName string, body []byte) error {
+	// The directory is world-traversable so uid runner can open the file.
+	// An already-present directory is not a failure: the next bake will
+	// create it from tmpfiles, and a retry of this write meets it again.
+	_ = cli.CreateInstanceFile(instanceName, imageIdentityDirectory, incus.InstanceFileArgs{
+		Content: bytes.NewReader(nil), UID: 0, GID: 0, Mode: 0o755, Type: "directory", WriteMode: "overwrite",
+	})
+	return cli.CreateInstanceFile(instanceName, imageIdentityPath, incus.InstanceFileArgs{
+		Content: bytes.NewReader(body), UID: 0, GID: 0, Mode: 0o644, Type: "file", WriteMode: "overwrite",
+	})
+}

internal/garmproviderincus/provider/image_identity_test.go

@@ -0,0 +1,86 @@
+package provider
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"testing"
+
+	incus "github.com/lxc/incus/v7/client"
+	"github.com/lxc/incus/v7/shared/api"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func expectImageIdentity(cli *MockIncusServer, instanceName, _ string) {
+	cli.On("CreateInstanceFile", instanceName, imageIdentityDirectory, mock.MatchedBy(func(args incus.InstanceFileArgs) bool {
+		return args.Type == "directory" && args.UID == 0 && args.GID == 0 && args.Mode == 0o755 && args.WriteMode == "overwrite"
+	})).Return(nil).Once()
+	// Content is not inspected here: sibling CreateInstanceFile matchers
+	// ReadAll the same bytes.Reader while testify diffs every expectation,
+	// which empties the reader before this matcher runs. The dedicated
+	// injectImageIdentity tests below prove the fingerprint bytes.
+	cli.On("CreateInstanceFile", instanceName, imageIdentityPath, mock.MatchedBy(func(args incus.InstanceFileArgs) bool {
+		return args.UID == 0 && args.GID == 0 && args.Mode == 0o644 && args.Type == "file" && args.WriteMode == "overwrite" && args.Content != nil
+	})).Return(nil).Once()
+}
+
+func TestInjectImageIdentityWritesWorldReadableFingerprint(t *testing.T) {
+	cli := new(MockIncusServer)
+	provider := newTestProvider(cli)
+	var got []byte
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityDirectory, mock.Anything).Return(nil).Once()
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityPath, mock.Anything).Run(func(args mock.Arguments) {
+		file := args.Get(2).(incus.InstanceFileArgs)
+		require.Equal(t, int64(0), file.UID)
+		require.Equal(t, int64(0), file.GID)
+		require.Equal(t, 0o644, int(file.Mode))
+		require.Equal(t, "file", file.Type)
+		body, err := io.ReadAll(file.Content)
+		require.NoError(t, err)
+		got = body
+	}).Return(nil).Once()
+
+	require.NoError(t, provider.injectImageIdentity(context.Background(), cli, "runner-test-instance", testImageDigest))
+	require.Equal(t, testImageDigest+"\n", string(got))
+	cli.AssertExpectations(t)
+}
+
+func TestInjectImageIdentityRefusesANonHexFingerprintWithoutTouchingTheGuest(t *testing.T) {
+	cli := new(MockIncusServer)
+	provider := newTestProvider(cli)
+
+	err := provider.injectImageIdentity(context.Background(), cli, "runner-test-instance", "not-a-fingerprint")
+	require.ErrorContains(t, err, "64-hex digest")
+	cli.AssertNotCalled(t, "CreateInstanceFile", mock.Anything, mock.Anything, mock.Anything)
+}
+
+func TestInjectImageIdentityRetriesUntilTheGuestAcceptsTheFile(t *testing.T) {
+	cli := new(MockIncusServer)
+	provider := newTestProvider(cli)
+	running := ownedInstance("runner-test-instance")
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityDirectory, mock.Anything).Return(nil).Twice()
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityPath, mock.Anything).Return(io.ErrUnexpectedEOF).Once()
+	cli.On("GetInstanceFull", "runner-test-instance").Return(running, "", nil).Once()
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityPath, mock.MatchedBy(func(args incus.InstanceFileArgs) bool {
+		content, err := io.ReadAll(args.Content)
+		return err == nil && args.Mode == 0o644 && bytes.Equal(bytes.TrimSpace(content), []byte(testImageDigest))
+	})).Return(nil).Once()
+
+	require.NoError(t, provider.injectImageIdentity(context.Background(), cli, "runner-test-instance", testImageDigest))
+	cli.AssertExpectations(t)
+}
+
+func TestInjectImageIdentityStopsWhenTheInstanceStops(t *testing.T) {
+	cli := new(MockIncusServer)
+	provider := newTestProvider(cli)
+	stopped := ownedInstance("runner-test-instance")
+	stopped.State = &api.InstanceState{Status: "Stopped"}
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityDirectory, mock.Anything).Return(nil).Once()
+	cli.On("CreateInstanceFile", "runner-test-instance", imageIdentityPath, mock.Anything).Return(io.ErrUnexpectedEOF).Once()
+	cli.On("GetInstanceFull", "runner-test-instance").Return(stopped, "", nil).Once()
+
+	err := provider.injectImageIdentity(context.Background(), cli, "runner-test-instance", testImageDigest)
+	require.ErrorContains(t, err, "instance stopped")
+	cli.AssertExpectations(t)
+}

internal/garmproviderincus/provider/incus.go

@@ -1251,6 +1251,9 @@ func (l *Incus) activateWarmInstance(
 		}
 		claimCommitted = false
 	}
+	if err := l.publishJobImageIdentity(ctx, cli, instance.Name, bootstrapParams.Flavor); err != nil {
+		return commonParams.ProviderInstance{}, err
+	}
 	if encodedJIT != "" {
 		if err := l.waitDirectJITAssignmentStarted(ctx, cli, instance.Name, bootstrapParams.Flavor); err != nil {
 			return commonParams.ProviderInstance{}, err
@@ -1315,6 +1318,9 @@ func (l *Incus) startColdDirectJIT(
 	if err := l.injectAssignment(ctx, cli, instanceName, warmAssignmentPath(bootstrap.Name), assignment); err != nil {
 		return commonParams.ProviderInstance{}, err
 	}
+	if err := l.publishJobImageIdentity(ctx, cli, instanceName, bootstrap.Flavor); err != nil {
+		return commonParams.ProviderInstance{}, err
+	}
 	claimCommitted = false
 	if err := l.admission.MarkCreated(ctx, instanceName); err != nil {
 		return commonParams.ProviderInstance{}, errors.Wrap(err, "recording created instance")
@@ -1375,6 +1381,9 @@ func (l *Incus) adoptColdDirectJIT(
 	switch {
 	case err == nil:
 		_ = content.Close()
+		if err := l.publishJobImageIdentity(ctx, cli, bootstrap.Name, bootstrap.Flavor); err != nil {
+			return commonParams.ProviderInstance{}, err
+		}
 		ret, err := l.waitInstanceHasIP(ctx, bootstrap.Name)
 		if err != nil {
 			return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching existing instance")
@@ -1615,6 +1624,13 @@ func (l *Incus) CreateInstance(ctx context.Context, bootstrapParams commonParams
 				return commonParams.ProviderInstance{}, err
 			}
 		}
+		cli, err := l.getCLI(ctx)
+		if err != nil {
+			return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching client")
+		}
+		if err := l.publishJobImageIdentity(ctx, cli, bootstrapParams.Name, bootstrapParams.Flavor); err != nil {
+			return commonParams.ProviderInstance{}, err
+		}
 		ret, err := l.waitInstanceHasIP(ctx, bootstrapParams.Name)
 		if err != nil {
 			return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching existing instance")
@@ -1715,6 +1731,13 @@ func (l *Incus) CreateInstance(ctx context.Context, bootstrapParams commonParams
 	if err := l.admission.MarkCreated(ctx, args.Name); err != nil {
 		return commonParams.ProviderInstance{}, errors.Wrap(err, "recording created instance")
 	}
+	cli, err := l.getCLI(ctx)
+	if err != nil {
+		return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching client")
+	}
+	if err := l.publishJobImageIdentity(ctx, cli, args.Name, bootstrapParams.Flavor); err != nil {
+		return commonParams.ProviderInstance{}, err
+	}
 
 	ret, err := l.waitInstanceHasIP(ctx, args.Name)
 	if err != nil {

internal/garmproviderincus/provider/incus_test.go

@@ -1077,6 +1077,7 @@ func TestCreateInstanceReturnsOnlineOwnedVM(t *testing.T) {
 	cli.On("UpdateInstanceState", "runner-test-instance", api.InstanceStatePut{Action: "start", Timeout: -1}, "").Return(op, nil).Once()
 	cli.On("GetInstanceFull", "runner-test-instance").Return((*api.InstanceFull)(nil), "", os.ErrNotExist).Once()
 	cli.On("GetInstanceFull", "runner-test-instance").Return(ownedInstance("runner-test-instance"), "", nil)
+	expectImageIdentity(cli, "runner-test-instance", testImageDigest)
 
 	got, err := provider.CreateInstance(context.Background(), validBootstrap())
 	require.NoError(t, err)
@@ -1111,6 +1112,7 @@ func TestCreateInstanceDeletesReservedWarmCapacityBeforeColdLaunch(t *testing.T)
 	cli.On("UpdateInstanceState", "runner-test-instance", api.InstanceStatePut{Action: "start", Timeout: -1}, "").
 		Return(createOperation, nil).Once()
 	cli.On("GetInstanceFull", "runner-test-instance").Return(ownedInstance("runner-test-instance"), "", nil)
+	expectImageIdentity(cli, "runner-test-instance", testImageDigest)
 
 	got, err := provider.CreateInstance(context.Background(), validBootstrap())
 	require.NoError(t, err)
@@ -1187,6 +1189,7 @@ func TestCreateInstanceClaimsRunningUnregisteredWarmVMWithoutColdCreate(t *testi
 			strings.Contains(string(content), base64.StdEncoding.EncodeToString(bootstrap.CACertBundle)) &&
 			!strings.Contains(string(content), string(bootstrap.CACertBundle)) && !strings.Contains(string(content), "set -x")
 	})).Return(nil).Once()
+	expectImageIdentity(cli, warm.Name, testImageDigest)
 	cli.On("GetInstanceFull", warm.Name).Return(activated, "", nil).Once()
 
 	got, err := provider.CreateInstance(context.Background(), bootstrap)
@@ -1196,7 +1199,7 @@ func TestCreateInstanceClaimsRunningUnregisteredWarmVMWithoutColdCreate(t *testi
 	require.True(t, warmControl.injected)
 	cli.AssertNotCalled(t, "CreateInstance", mock.Anything)
 	cli.AssertNotCalled(t, "GetImage", mock.Anything)
-	cli.AssertNumberOfCalls(t, "CreateInstanceFile", 1)
+	cli.AssertNumberOfCalls(t, "CreateInstanceFile", 3)
 	cli.AssertExpectations(t)
 }
 
@@ -1225,6 +1228,7 @@ func TestCreateInstanceDirectJITClaimBypassesMetadataInstaller(t *testing.T) {
 			strings.Contains(string(content), encoded) && strings.Contains(string(content), "--jitconfig") &&
 			!strings.Contains(string(content), bootstrap.InstanceToken) && !strings.Contains(string(content), expectedMetadataURL)
 	})).Return(nil).Once()
+	expectImageIdentity(cli, warm.Name, testImageDigest)
 	cli.On("GetInstanceFile", warm.Name, directJITPhasePath).Return(
 		io.NopCloser(strings.NewReader("{\"schema_version\":1,\"phase\":\"assignment-script-started\",\"unix_ns\":1786327000000000000}\n")),
 		&incus.InstanceFileResponse{Type: "file", UID: 1001, GID: 1002, Mode: 0o600}, nil,
@@ -1252,20 +1256,21 @@ func TestCreateInstanceRetryAdoptsInjectedWarmClaimWithoutReinjecting(t *testing
 	activated := ownedInstance(warmControl.claim.InstanceName)
 	cli.On("GetInstanceFull", "runner-test-instance").Return((*api.InstanceFull)(nil), "", os.ErrNotExist).Once()
 	cli.On("GetInstanceFull", activated.Name).Return(activated, "warm-etag", nil).Twice()
+	expectImageIdentity(cli, activated.Name, testImageDigest)
 
 	got, err := provider.CreateInstance(context.Background(), validBootstrap())
 	require.NoError(t, err)
 	require.Equal(t, "runner-test-instance", got.Name)
 	require.Equal(t, activated.Name, got.ProviderID)
 	cli.AssertNotCalled(t, "UpdateInstance", mock.Anything, mock.Anything, mock.Anything)
-	cli.AssertNotCalled(t, "CreateInstanceFile", mock.Anything, mock.Anything, mock.Anything)
 }
 
 func TestCreateInstanceIdempotentlyAdoptsMatchingOwnedVM(t *testing.T) {
 	cli := new(MockIncusServer)
 	provider := newTestProvider(cli)
 	instance := ownedInstance("runner-test-instance")
 	cli.On("GetInstanceFull", instance.Name).Return(instance, "", nil).Twice()
+	expectImageIdentity(cli, instance.Name, testImageDigest)
 
 	got, err := provider.CreateInstance(context.Background(), validBootstrap())
 	require.NoError(t, err)
@@ -1286,6 +1291,7 @@ func TestCreateInstanceRetryAdoptsVMCreatedBeforeAmbiguousOperationTimeout(t *te
 		Return((*api.InstanceFull)(nil), "", os.ErrNotExist).Once()
 	cli.On("CreateInstance", mock.Anything).Return(createOperation, nil).Once()
 	cli.On("GetInstanceFull", instance.Name).Return(instance, "", nil).Twice()
+	expectImageIdentity(cli, instance.Name, testImageDigest)
 
 	_, err := provider.CreateInstance(context.Background(), validBootstrap())
 	require.ErrorContains(t, err, "waiting for instance creation")
@@ -1588,6 +1594,7 @@ func TestCreateInstanceDirectJITColdInjectsTheAssignmentInsteadOfCloudInit(t *te
 			strings.Contains(string(content), encoded) && strings.Contains(string(content), "--jitconfig") &&
 			!strings.Contains(string(content), bootstrap.InstanceToken) && !strings.Contains(string(content), expectedMetadataURL)
 	})).Return(nil).Once()
+	expectImageIdentity(cli, "runner-test-instance", testImageDigest)
 	cli.On("GetInstanceFile", "runner-test-instance", directJITPhasePath).Return(
 		io.NopCloser(strings.NewReader("{\"schema_version\":1,\"phase\":\"assignment-script-started\",\"unix_ns\":1786327000000000000}\n")),
 		&incus.InstanceFileResponse{Type: "file", UID: 1001, GID: 1002, Mode: 0o600}, nil,
@@ -1612,12 +1619,12 @@ func TestCreateInstanceDirectJITColdRetryAdoptsAStartedWorkerWithoutReinjecting(
 		io.NopCloser(strings.NewReader("{\"schema_version\":1,\"phase\":\"assignment-script-started\",\"unix_ns\":1786327000000000000}\n")),
 		&incus.InstanceFileResponse{Type: "file", UID: 1001, GID: 1002, Mode: 0o600}, nil,
 	).Once()
+	expectImageIdentity(cli, "runner-test-instance", testImageDigest)
 
 	got, err := provider.CreateInstance(context.Background(), directJITBootstrap(t))
 	require.NoError(t, err)
 	require.Equal(t, "runner-test-instance", got.ProviderID)
 	cli.AssertNotCalled(t, "CreateInstance", mock.Anything)
-	cli.AssertNotCalled(t, "CreateInstanceFile", mock.Anything, mock.Anything, mock.Anything)
 	cli.AssertExpectations(t)
 }
 
@@ -1634,6 +1641,7 @@ func TestCreateInstanceDirectJITColdRetryDeliversAMissingAssignment(t *testing.T
 		content, err := io.ReadAll(args.Content)
 		return err == nil && strings.Contains(string(content), encoded)
 	})).Return(nil).Once()
+	expectImageIdentity(cli, "runner-test-instance", testImageDigest)
 	cli.On("GetInstanceFile", "runner-test-instance", directJITPhasePath).Return(
 		io.NopCloser(strings.NewReader("{\"schema_version\":1,\"phase\":\"assignment-script-started\",\"unix_ns\":1786327000000000000}\n")),
 		&incus.InstanceFileResponse{Type: "file", UID: 1001, GID: 1002, Mode: 0o600}, nil,