feat!: new lockfile format with GitHub release support via Packslip (PoC)

.github/workflows/wasm.yml

@@ -66,6 +66,11 @@ jobs:
           go-version-file: "./go.mod"
           cache: true
           cache-dependency-path: go.sum
+      - name: Compile Packslip source resolution for WebAssembly
+        run: go build ./internal/packslip ./internal/packslipverify ./internal/sourceresolution
+        env:
+          GOOS: js
+          GOARCH: wasm
       - name: Node version
         run: node --version
       - name: Build @columnar-tech/dbc-wasm

.typos.toml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 [files]
-extend-exclude = ["columnar.pubkey", "go.mod", "go.sum", "pixi.lock"]
+extend-exclude = ["columnar.pubkey", "go.mod", "go.sum", "pixi.lock", "internal/packslipverify/testdata/bundle-provenance.json"]
 
 [type.fo]
 extend-glob = ["*.go", ".goreleaser.yaml"]

client_methods.go

@@ -137,6 +137,9 @@ func (c *Client) getDriverListFromIndex(ctx context.Context, index *Registry) ([
 	for i := range drivers.Drivers {
 		drivers.Drivers[i].Registry = index
 	}
+	if err := validateRegistryMetadata(drivers.Drivers); err != nil {
+		return nil, fmt.Errorf("failed to parse driver registry index: %w", err)
+	}
 
 	result := drivers.Drivers
 	sort.Slice(result, func(i, j int) bool {
@@ -271,25 +274,56 @@ func (c *Client) Install(ctx context.Context, cfg config.Config, driverName stri
 	if err != nil {
 		return nil, fmt.Errorf("failed to get package for driver %s: %w", driverName, err)
 	}
+	expected, err := expectedRegistryPackageMetadata(pkg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to install driver %s: %w", driverName, err)
+	}
 
 	f, err := c.downloadPackage(ctx, pkg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to download driver %s: %w", driverName, err)
 	}
+	defer f.Close()
 	defer os.RemoveAll(filepath.Dir(f.Name()))
 
-	manifest, err := config.InstallDriver(cfg, driverName, f)
+	manifest, err := config.InstallPackage(cfg, expected.ID, f, expected, config.InstallOptions{
+		Verify: func(stagingDir string, manifest config.Manifest) error {
+			return VerifyPackageSignature(stagingDir, manifest)
+		},
+	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to install driver %s: %w", driverName, err)
 	}
 
-	if err := config.CreateManifest(cfg, manifest.DriverInfo); err != nil {
-		return nil, fmt.Errorf("failed to create manifest for driver %s: %w", driverName, err)
-	}
-
 	return &manifest, nil
 }
 
+func expectedRegistryPackageMetadata(pkg PkgInfo) (config.ExpectedPackageMetadata, error) {
+	hasHash := pkg.ArtifactHash != ""
+	if pkg.Version == nil {
+		return config.ExpectedPackageMetadata{}, errors.New("registry package metadata is missing its version")
+	}
+	if strings.TrimSpace(pkg.PlatformTuple) == "" {
+		return config.ExpectedPackageMetadata{}, errors.New("registry package metadata is missing its platform")
+	}
+	if pkg.Driver.Registry == nil || pkg.Driver.Registry.BaseURL == nil {
+		return config.ExpectedPackageMetadata{}, errors.New("registry package metadata is missing its source identity")
+	}
+	if strings.TrimSpace(pkg.Driver.Path) == "" {
+		return config.ExpectedPackageMetadata{}, errors.New("registry package metadata is missing its driver ID")
+	}
+	expected := config.ExpectedPackageMetadata{
+		ID:         pkg.Driver.Path,
+		Version:    pkg.Version.String(),
+		Platform:   pkg.PlatformTuple,
+		SourceType: "registry", SourceIdentity: pkg.Driver.Registry.BaseURL.String(),
+	}
+	if hasHash {
+		expected.ArchiveHash = pkg.ArtifactHash
+	}
+	return expected, nil
+}
+
 // Uninstall uninstalls a driver with the given name from the specified configuration.
 func (c *Client) Uninstall(cfg config.Config, driverName string) error {
 	di, err := config.GetDriver(cfg, driverName)

client_methods_test.go

@@ -15,6 +15,9 @@
 package dbc_test
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -31,6 +34,69 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func readClientLegacyArchive(t *testing.T) []byte {
+	t.Helper()
+	archive, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test-driver-1.1.tar.gz"))
+	require.NoError(t, err)
+	return archive
+}
+
+func newArchiveMetadataInstallServer(t *testing.T, archive []byte, metadata string) *httptest.Server {
+	t.Helper()
+	digest := sha256.Sum256(archive)
+	index := fmt.Sprintf(`drivers:
+  - name: Test Driver
+    path: test-driver-1
+    pkginfo:
+      - version: v1.1.0
+        packages:
+          - platform: %s
+            url: package.tar.gz
+`, config.PlatformTuple())
+	if metadata == "complete" {
+		index += fmt.Sprintf("            hash: %q\n            future_metadata: ignored\n", "sha256:"+hex.EncodeToString(digest[:]))
+	} else if metadata == "hash-only" {
+		index += fmt.Sprintf("            hash: %q\n", "sha256:"+hex.EncodeToString(digest[:]))
+	}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/index.yaml":
+			_, _ = w.Write([]byte(index))
+		case "/package.tar.gz":
+			_, _ = w.Write(archive)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(server.Close)
+	return server
+}
+
+func newLegacyInstallServer(t *testing.T, archive []byte) *httptest.Server {
+	t.Helper()
+	index := fmt.Sprintf(`drivers:
+  - name: Test Driver
+    path: test-driver-1
+    pkginfo:
+      - version: v1.1.0
+        packages:
+          - platform: %s
+            url: package.tar.gz
+`, config.PlatformTuple())
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/index.yaml":
+			_, _ = w.Write([]byte(index))
+		case "/package.tar.gz":
+			_, _ = w.Write(archive)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(server.Close)
+	return server
+}
+
 func newTestClientForServer(t *testing.T, serverURL string) *dbc.Client {
 	t.Helper()
 	c, err := dbc.NewClient(
@@ -47,7 +113,7 @@ func newInstallTestServer(t *testing.T) *httptest.Server {
 	indexData, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test_index.yaml"))
 	require.NoError(t, err)
 
-	tarballData, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test-driver-1.tar.gz"))
+	tarballData, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test-driver-1.1.tar.gz"))
 	require.NoError(t, err)
 
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -133,6 +199,132 @@ func TestClientInstall(t *testing.T) {
 	})
 }
 
+func TestClientInstallSignatureFailurePreservesExistingInstallation(t *testing.T) {
+	oldArchive, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test-driver-1.tar.gz"))
+	require.NoError(t, err)
+	badArchive, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test-driver-no-sig.tar.gz"))
+	require.NoError(t, err)
+	archivePath := filepath.Join(t.TempDir(), "old.tar.gz")
+	require.NoError(t, os.WriteFile(archivePath, oldArchive, 0o600))
+	oldFile, err := os.Open(archivePath)
+	require.NoError(t, err)
+	defer oldFile.Close()
+
+	root := t.TempDir()
+	cfg := config.Config{Level: config.ConfigEnv, Location: root}
+	oldExpected := config.ExpectedPackageMetadata{
+		ID: "test-driver-1", Version: "1.0.0", Platform: config.PlatformTuple(),
+		SourceType: "registry", SourceIdentity: "https://registry.example.test",
+	}
+	oldManifest, err := config.InstallPackage(cfg, "test-driver-1", oldFile, oldExpected, config.InstallOptions{
+		Verify: func(stagingDir string, manifest config.Manifest) error {
+			return dbc.VerifyPackageSignature(stagingDir, manifest)
+		},
+	})
+	require.NoError(t, err)
+	oldManifestBytes, err := os.ReadFile(filepath.Join(root, "test-driver-1.toml"))
+	require.NoError(t, err)
+	oldLibraryPath := oldManifest.Driver.Shared.Get(config.PlatformTuple())
+	oldLibraryBytes, err := os.ReadFile(oldLibraryPath)
+	require.NoError(t, err)
+
+	server := newLegacyInstallServer(t, badArchive)
+	client := newTestClientForServer(t, server.URL)
+	_, err = client.Install(t.Context(), cfg, "test-driver-1")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "signature file 'test-driver-1-not-valid.so.sig' for driver is missing")
+
+	manifestBytes, err := os.ReadFile(filepath.Join(root, "test-driver-1.toml"))
+	require.NoError(t, err)
+	assert.Equal(t, oldManifestBytes, manifestBytes)
+	libraryBytes, err := os.ReadFile(oldLibraryPath)
+	require.NoError(t, err)
+	assert.Equal(t, oldLibraryBytes, libraryBytes)
+}
+
+func TestClientInstallLegacyMetadataMismatchPreservesExistingInstallation(t *testing.T) {
+	archive, err := os.ReadFile(filepath.Join("cmd", "dbc", "testdata", "test-driver-1.tar.gz"))
+	require.NoError(t, err)
+	archivePath := filepath.Join(t.TempDir(), "old.tar.gz")
+	require.NoError(t, os.WriteFile(archivePath, archive, 0o600))
+	oldFile, err := os.Open(archivePath)
+	require.NoError(t, err)
+	defer oldFile.Close()
+
+	root := t.TempDir()
+	cfg := config.Config{Level: config.ConfigEnv, Location: root}
+	oldManifest, err := config.InstallPackage(cfg, "test-driver-1", oldFile, config.ExpectedPackageMetadata{
+		ID: "test-driver-1", Version: "1.0.0", Platform: config.PlatformTuple(),
+		SourceType: "registry", SourceIdentity: "https://registry.example.test",
+	}, config.InstallOptions{
+		Verify: func(stagingDir string, manifest config.Manifest) error {
+			return dbc.VerifyPackageSignature(stagingDir, manifest)
+		},
+	})
+	require.NoError(t, err)
+	oldManifestBytes, err := os.ReadFile(filepath.Join(root, "test-driver-1.toml"))
+	require.NoError(t, err)
+	oldLibraryPath := oldManifest.Driver.Shared.Get(config.PlatformTuple())
+	oldLibraryBytes, err := os.ReadFile(oldLibraryPath)
+	require.NoError(t, err)
+
+	server := newLegacyInstallServer(t, archive)
+	client := newTestClientForServer(t, server.URL)
+	_, err = client.Install(t.Context(), cfg, "test-driver-1")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `package version mismatch: archive declares "1.0.0", expected "1.1.0"`)
+
+	manifestBytes, err := os.ReadFile(filepath.Join(root, "test-driver-1.toml"))
+	require.NoError(t, err)
+	assert.Equal(t, oldManifestBytes, manifestBytes)
+	libraryBytes, err := os.ReadFile(oldLibraryPath)
+	require.NoError(t, err)
+	assert.Equal(t, oldLibraryBytes, libraryBytes)
+}
+
+func TestClientInstallLegacyPackageUsesRegistryMetadata(t *testing.T) {
+	archive := readClientLegacyArchive(t)
+	srv := newArchiveMetadataInstallServer(t, archive, "complete")
+	c := newTestClientForServer(t, srv.URL)
+	root := t.TempDir()
+	cfg := config.Config{Level: config.ConfigEnv, Location: root}
+
+	manifest, err := c.Install(t.Context(), cfg, "test-driver-1")
+	require.NoError(t, err)
+	require.NotNil(t, manifest)
+
+	var receipt config.InstallReceipt
+	receiptBytes, err := os.ReadFile(filepath.Join(filepath.Dir(manifest.DriverInfo.Driver.Shared.Get(config.PlatformTuple())), "dbc-install-receipt.json"))
+	require.NoError(t, err)
+	require.NoError(t, json.Unmarshal(receiptBytes, &receipt))
+	assert.Equal(t, "registry", receipt.SourceType)
+	assert.Equal(t, srv.URL, receipt.SourceIdentity)
+	assert.NotEmpty(t, receipt.ArchiveHash)
+	assert.Positive(t, receipt.ArchiveSize)
+
+	runtimeManifest, err := os.ReadFile(filepath.Join(root, "test-driver-1.toml"))
+	require.NoError(t, err)
+	assert.Contains(t, string(runtimeManifest), "manifest_version = 1")
+}
+
+func TestClientInstallAcceptsHashOnlyRegistryArchiveMetadata(t *testing.T) {
+	archive := readClientLegacyArchive(t)
+	srv := newArchiveMetadataInstallServer(t, archive, "hash-only")
+	c := newTestClientForServer(t, srv.URL)
+	root := t.TempDir()
+	cfg := config.Config{Level: config.ConfigEnv, Location: root}
+
+	manifest, err := c.Install(t.Context(), cfg, "test-driver-1")
+	require.NoError(t, err)
+	require.NotNil(t, manifest)
+	var receipt config.InstallReceipt
+	receiptBytes, err := os.ReadFile(filepath.Join(filepath.Dir(manifest.DriverInfo.Driver.Shared.Get(config.PlatformTuple())), "dbc-install-receipt.json"))
+	require.NoError(t, err)
+	require.NoError(t, json.Unmarshal(receiptBytes, &receipt))
+	assert.NotEmpty(t, receipt.ArchiveHash)
+	assert.Positive(t, receipt.ArchiveSize, "receipt retains the measured archive size")
+}
+
 func TestClientUninstall(t *testing.T) {
 	srv := newInstallTestServer(t)
 	c := newTestClientForServer(t, srv.URL)