diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index 8272ee94bc..f16d01f456 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -16,7 +16,7 @@ If you believe you have found a new, distinct issue, report it to the Obol Secur * **Legacy lock formats are retained for compatibility.** Charon must keep loading the existing cluster locks of running clusters. Some lock formats from before v1.7 carry fewer signatures than current ones. * **`--no-verify` is out of scope.** This flag disables cluster definition and lock verification. Findings that depend on an operator running with it are not eligible. * **Operators control their own node and its inputs.** Operators choose their own beacon node, validator client, configuration, and cluster lock file. Findings that depend on an operator running a lock, configuration, or binary they did not produce or verify themselves are not eligible. -* **Access to the operator's host is out of scope.** Findings that require reading files, archives, logs, or command output on the operator's machine assume the host is already compromised and are not eligible. Examples include the file permissions of cluster or DKG archives created with `--zipped`, and secrets visible in local logs or test output. Since v1.11.3, Charon redacts credentials and API keys in configured endpoint URLs before logging them. +* **Access to the operator's host is out of scope.** Findings that require reading files, archives, logs, or command output on the operator's machine assume the host is already compromised and are not eligible. Examples include secrets visible in local logs or test output. Since v2.0.0, cluster and DKG archives created with `--zipped` are readable only by their owner. Since v1.11.3, Charon redacts credentials and API keys in configured endpoint URLs before logging them. ## Cluster Definition & Lock diff --git a/app/utils.go b/app/utils.go index a36321961e..3a73468c30 100644 --- a/app/utils.go +++ b/app/utils.go @@ -4,7 +4,6 @@ package app import ( "archive/tar" - "bytes" "compress/gzip" "encoding/hex" "io" @@ -27,20 +26,32 @@ func Hex7(input []byte) string { } // BundleOutput archives targetDir into a gzipped tarball named filename in targetDir. -// After successfully creating the archive, it deletes the original files from disk. -func BundleOutput(targetDir string, filename string) error { - // Create output file - file, err := os.Create(filepath.Join(targetDir, filename)) +// The archive contains private key material, so it is only readable and writable by its owner. +// It is written to a temporary file and renamed into place once complete, so filename is +// never a partial archive. The original files are deleted only once the archive is in place, +// so a failure before that leaves them untouched. +func BundleOutput(targetDir string, filename string) (err error) { + archivePath := filepath.Join(targetDir, filename) + tempPrefix := filename + ".tmp-" + + // CreateTemp creates the tempFile with mode 0600. + tempFile, err := os.CreateTemp(targetDir, tempPrefix+"*") if err != nil { return errors.Wrap(err, "create .tar.gz file") } - defer file.Close() - gzw := gzip.NewWriter(file) - defer gzw.Close() + // CreateTemp does not clean its path while filepath.Walk does, e.g. for the default "./" directory. + tempPath := filepath.Clean(tempFile.Name()) + defer func() { + if err != nil { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + } + }() + + gzw := gzip.NewWriter(tempFile) tw := tar.NewWriter(gzw) - defer tw.Close() // Files to be deleted dirsToDelete := make([]string, 0) @@ -61,8 +72,15 @@ func BundleOutput(targetDir string, filename string) error { return nil } - // Don't delete output file - if path == file.Name() { + // Don't archive or delete the output file, nor an archive left by a previous run (it is replaced). + if path == tempPath || path == archivePath { + return nil + } + + // Don't archive a temporary file left by an interrupted run, but do delete it. + if strings.HasPrefix(info.Name(), tempPrefix) { + filesToDelete = append(filesToDelete, path) + return nil } @@ -100,168 +118,41 @@ func BundleOutput(targetDir string, filename string) error { return errors.Wrap(err, "filepath walk") } - // Delete files/folders - for _, path := range filesToDelete { - err := os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, "remove file", z.Str("filename", path)) - } + // Flush the archive to disk and move it into place before deleting the originals, + // so a failed write never loses them. + if err := tw.Close(); err != nil { + return errors.Wrap(err, "close tar writer") } - for _, path := range dirsToDelete { - err := os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, "remove dir", z.Str("filename", path)) - } + if err := gzw.Close(); err != nil { + return errors.Wrap(err, "close gzip writer") } - return nil -} - -// ExtractArchive extracts a .tar.gz archive to the target directory. -func ExtractArchive(archivePath, targetDir string) error { - file, err := os.Open(archivePath) - if err != nil { - return errors.Wrap(err, "open archive file") + if err := tempFile.Sync(); err != nil { + return errors.Wrap(err, "sync .tar.gz file") } - defer file.Close() - gzr, err := gzip.NewReader(file) - if err != nil { - return errors.Wrap(err, "create gzip reader") + if err := tempFile.Close(); err != nil { + return errors.Wrap(err, "close .tar.gz file") } - defer gzr.Close() - - tr := tar.NewReader(gzr) - - for { - header, err := tr.Next() - if errors.Is(err, io.EOF) { - return nil // End of archive - } - - if err != nil { - return errors.Wrap(err, "tar read error") - } - - cleanName := filepath.Clean(header.Name) - // Disallow absolute paths and paths with ".." as first element - if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) || strings.Contains(cleanName, ".."+string(os.PathSeparator)) { - return errors.New("invalid archive entry path", z.Str("path", header.Name)) - } - - target := filepath.Join(targetDir, cleanName) - - absTargetDir, err := filepath.Abs(targetDir) - if err != nil { - return errors.Wrap(err, "get absolute target dir") - } - - absTarget, err := filepath.Abs(target) - if err != nil { - return errors.Wrap(err, "get absolute target path") - } - - if !strings.HasPrefix(absTarget, absTargetDir+string(os.PathSeparator)) && absTarget != absTargetDir { - return errors.New("archive entry path escapes target directory", z.Str("path", header.Name)) - } - - switch header.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(target, header.FileInfo().Mode()); err != nil { - return errors.Wrap(err, "create directory") - } - case tar.TypeReg: - // Check file size to prevent decompression bombs - const maxFileSize = 100 * 1024 * 1024 // 100MB limit per file - if header.Size > maxFileSize { - return errors.New("file too large in archive", z.Str("path", header.Name), z.I64("size", header.Size)) - } - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { - return errors.Wrap(err, "create parent directory") - } - - f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, header.FileInfo().Mode()) - if err != nil { - return errors.Wrap(err, "create file") - } - - // Use limited reader to prevent decompression bombs - limitedReader := io.LimitReader(tr, maxFileSize) - - _, err = io.Copy(f, limitedReader) - if err != nil { - f.Close() - return errors.Wrap(err, "copy file contents") - } - - if err := f.Close(); err != nil { - return errors.Wrap(err, "close file") - } - default: - // Skip other types (symlinks, etc.) - } + if err := os.Rename(tempFile.Name(), archivePath); err != nil { + return errors.Wrap(err, "rename .tar.gz file") } -} - -// CompareDirectories recursively compares two directories and their contents. -func CompareDirectories(originalDir, extractedDir string) error { - err := filepath.Walk(originalDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(originalDir, path) - if err != nil { - return errors.Wrap(err, "get relative path") - } - - extractedPath := filepath.Join(extractedDir, relPath) - if info.IsDir() { - // Check if directory exists in extracted content - extractedInfo, err := os.Stat(extractedPath) - if err != nil { - return errors.Wrap(err, "directory should exist in extracted content", z.Str("path", relPath)) - } - - if !extractedInfo.IsDir() { - return errors.New("path is not a directory", z.Str("path", relPath)) - } - - return nil - } - - // Compare file contents - extractedInfo, err := os.Stat(extractedPath) - if err != nil { - return errors.Wrap(err, "file should exist in extracted content", z.Str("path", relPath)) - } - - if info.Size() != extractedInfo.Size() { - return errors.New("file size mismatch", z.Str("path", relPath)) - } - - // Read and compare file contents - originalContent, err := os.ReadFile(path) //nolint:gosec // no TOCTOU condition as CompareDirectories is used only in CLI non-concurrent context. + // Delete files/folders + for _, path := range filesToDelete { + err := os.RemoveAll(path) if err != nil { - return errors.Wrap(err, "read original file", z.Str("path", path)) + return errors.Wrap(err, "remove file", z.Str("filename", path)) } + } - extractedContent, err := os.ReadFile(extractedPath) + for _, path := range dirsToDelete { + err := os.RemoveAll(path) if err != nil { - return errors.Wrap(err, "read extracted file", z.Str("path", extractedPath)) - } - - if !bytes.Equal(originalContent, extractedContent) { - return errors.New("file content mismatch", z.Str("path", relPath)) + return errors.Wrap(err, "remove dir", z.Str("filename", path)) } - - return nil - }) - if err != nil { - return errors.Wrap(err, "walk directory tree") } return nil diff --git a/app/utils_test.go b/app/utils_test.go index b2342a7870..f3d7861714 100644 --- a/app/utils_test.go +++ b/app/utils_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/obolnetwork/charon/app" + "github.com/obolnetwork/charon/testutil" ) func TestHex7(t *testing.T) { @@ -67,10 +68,11 @@ func TestBundleOutput(t *testing.T) { err := app.BundleOutput(testDir, archiveName) require.NoError(t, err, "BundleOutput should succeed") - // Verify that the archive file exists + // Verify that the archive file exists and is only accessible by its owner archivePath := filepath.Join(testDir, archiveName) - _, err = os.Stat(archivePath) + info, err := os.Stat(archivePath) require.NoError(t, err, "Archive file should exist") + require.Equal(t, os.FileMode(0o600), info.Mode().Perm(), "Archive should be owner read/write only") // Verify that original files are deleted (except the archive) entries, err := os.ReadDir(testDir) @@ -80,184 +82,120 @@ func TestBundleOutput(t *testing.T) { // Extract the archive to a new directory extractDir := t.TempDir() - err = app.ExtractArchive(archivePath, extractDir) + err = testutil.ExtractArchive(archivePath, extractDir) require.NoError(t, err, "Archive extraction should succeed") // Compare the extracted content with the original backup - err = app.CompareDirectories(backupDir, extractDir) + err = testutil.CompareDirectories(backupDir, extractDir) require.NoError(t, err, "Extracted content should match original") } -func TestCompareDirectories(t *testing.T) { - tests := []struct { - name string - setupDirs func(t *testing.T) (dir1, dir2 string) - expectError bool - errorMsg string - }{ - { - name: "identical_directories", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - testFiles := map[string]string{ - "file1.txt": "content1", - "nested/file2.json": `{"key": "value"}`, - "nested/deep/file3.md": "# Header\nContent", - "binary.bin": "\x00\x01\x02\x03", - "special_chars_äöü.txt": "Special characters: äöüß", - } - - for relPath, content := range testFiles { - fullPath := filepath.Join(dir1, relPath) - require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) - require.NoError(t, os.WriteFile(fullPath, []byte(content), 0o644)) - } - - dir2 := t.TempDir() - require.NoError(t, os.CopyFS(dir2, os.DirFS(dir1))) - - return dir1, dir2 - }, - expectError: false, - }, - { - name: "missing_file", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir1, "file.txt"), []byte("content"), 0o644)) - - return dir1, dir2 - }, - expectError: true, - errorMsg: "file should exist in extracted content", - }, - { - name: "different_content", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir1, "file.txt"), []byte("content1"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir2, "file.txt"), []byte("content2"), 0o644)) - - return dir1, dir2 - }, - expectError: true, - errorMsg: "file content mismatch", - }, - { - name: "different_sizes", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir1, "file.txt"), []byte("short"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir2, "file.txt"), []byte("much longer content"), 0o644)) - - return dir1, dir2 - }, - expectError: true, - errorMsg: "file size mismatch", - }, - { - name: "missing_directory", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - nestedDir := filepath.Join(dir1, "nested", "deep") - require.NoError(t, os.MkdirAll(nestedDir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(nestedDir, "file.txt"), []byte("content"), 0o644)) - - return dir1, dir2 - }, - expectError: true, - errorMsg: "directory should exist in extracted content", - }, - { - name: "file_vs_directory", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir1, "item"), []byte("content"), 0o644)) - require.NoError(t, os.MkdirAll(filepath.Join(dir2, "item"), 0o755)) - - return dir1, dir2 - }, - expectError: true, - errorMsg: "file size mismatch", - }, - { - name: "directory_vs_file", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir1, "item"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir2, "item"), []byte("content"), 0o644)) - - return dir1, dir2 - }, - expectError: true, - errorMsg: "path is not a directory", - }, - { - name: "complex_structure", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - dir1 := t.TempDir() - dir2 := t.TempDir() - files := map[string]string{ - "root.txt": "root content", - "validator_keys/keystore-1.json": `{"crypto": {"cipher": "test"}}`, - "validator_keys/keystore-2.json": `{"crypto": {"cipher": "test"}}`, - "nested/level1/level2/deep.yaml": "key: value\narray:\n - item1\n - item2", - "cluster-lock.json": `{"lock_hash": "0xabc"}`, - "deposit_data.json": `[{"pubkey": "0x123"}]`, - "empty_dir/placeholder.txt": "", - "binary_data.bin": "\x00\x01\x02\x03\xFF\xFE\xFD", - } - - for relPath, content := range files { - for _, dir := range []string{dir1, dir2} { - fullPath := filepath.Join(dir, relPath) - require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) - require.NoError(t, os.WriteFile(fullPath, []byte(content), 0o644)) - } - } - - return dir1, dir2 - }, - expectError: false, - }, - { - name: "empty_directories", - setupDirs: func(t *testing.T) (string, string) { - t.Helper() - return t.TempDir(), t.TempDir() - }, - expectError: false, - }, - } +func TestBundleOutputExistingArchive(t *testing.T) { + testDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(testDir, "charon-enr-private-key"), []byte("secret"), 0o600)) + + // A world-readable archive left over from a previous run must be restricted too. + archivePath := filepath.Join(testDir, "test_bundle.tar.gz") + require.NoError(t, os.WriteFile(archivePath, []byte("stale"), 0o644)) + + require.NoError(t, app.BundleOutput(testDir, "test_bundle.tar.gz")) + + info, err := os.Stat(archivePath) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm(), "Archive should be owner read/write only") + + // Only the archive is left: no temporary file, no original. + entries, err := os.ReadDir(testDir) + require.NoError(t, err) + require.Len(t, entries, 1) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dir1, dir2 := tt.setupDirs(t) - err := app.CompareDirectories(dir1, dir2) + // The stale archive is replaced, not archived. + extractDir := t.TempDir() + require.NoError(t, testutil.ExtractArchive(archivePath, extractDir)) + entries, err = os.ReadDir(extractDir) + require.NoError(t, err) + require.Len(t, entries, 1) - if tt.expectError { - require.Error(t, err, "Expected error for test case: %s", tt.name) + b, err := os.ReadFile(filepath.Join(extractDir, "charon-enr-private-key")) + require.NoError(t, err) + require.Equal(t, "secret", string(b)) +} - if tt.errorMsg != "" { - require.Contains(t, err.Error(), tt.errorMsg, "Error message should contain expected text") - } - } else { - require.NoError(t, err, "Expected no error for test case: %s", tt.name) - } +func TestBundleOutputRelativeDir(t *testing.T) { + // The CLI defaults are relative: "./" for create cluster and ".charon" for dkg. + for _, dir := range []string{"./", ".", "sub", "sub/"} { + t.Run(dir, func(t *testing.T) { + t.Chdir(t.TempDir()) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "charon-enr-private-key"), []byte("secret"), 0o600)) + + // A temporary file left by an interrupted run is neither archived nor kept. + stale := filepath.Join(dir, "test_bundle.tar.gz.tmp-123") + require.NoError(t, os.WriteFile(stale, []byte("partial"), 0o600)) + + require.NoError(t, app.BundleOutput(dir, "test_bundle.tar.gz")) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "test_bundle.tar.gz", entries[0].Name()) + + extractDir := t.TempDir() + require.NoError(t, testutil.ExtractArchive(filepath.Join(dir, "test_bundle.tar.gz"), extractDir)) + entries, err = os.ReadDir(extractDir) + require.NoError(t, err) + require.Len(t, entries, 1) + + b, err := os.ReadFile(filepath.Join(extractDir, "charon-enr-private-key")) + require.NoError(t, err) + require.Equal(t, "secret", string(b)) }) } } + +func TestBundleOutputFailureKeepsOriginals(t *testing.T) { + testDir := t.TempDir() + keyPath := filepath.Join(testDir, "charon-enr-private-key") + require.NoError(t, os.WriteFile(keyPath, []byte("secret"), 0o600)) + + // An unreadable file makes archiving fail part way through. + unreadable := filepath.Join(testDir, "unreadable") + require.NoError(t, os.WriteFile(unreadable, []byte("x"), 0o000)) + + if _, err := os.ReadFile(unreadable); err == nil { + t.Skip("running as a user that ignores file permissions") + } + + require.Error(t, app.BundleOutput(testDir, "test_bundle.tar.gz")) + + // The originals are kept and no partial or temporary archive is left behind. + b, err := os.ReadFile(keyPath) + require.NoError(t, err) + require.Equal(t, "secret", string(b)) + + entries, err := os.ReadDir(testDir) + require.NoError(t, err) + require.Len(t, entries, 2) + + // Once the cause is fixed, a rerun succeeds and bundles exactly the originals. + require.NoError(t, os.Chmod(unreadable, 0o600)) + require.NoError(t, app.BundleOutput(testDir, "test_bundle.tar.gz")) + + entries, err = os.ReadDir(testDir) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "test_bundle.tar.gz", entries[0].Name()) + + extractDir := t.TempDir() + require.NoError(t, testutil.ExtractArchive(filepath.Join(testDir, "test_bundle.tar.gz"), extractDir)) + entries, err = os.ReadDir(extractDir) + require.NoError(t, err) + require.Len(t, entries, 2) + + b, err = os.ReadFile(filepath.Join(extractDir, "charon-enr-private-key")) + require.NoError(t, err) + require.Equal(t, "secret", string(b)) + b, err = os.ReadFile(filepath.Join(extractDir, "unreadable")) + require.NoError(t, err) + require.Equal(t, "x", string(b)) +} diff --git a/cmd/createcluster_internal_test.go b/cmd/createcluster_internal_test.go index ebb8aded6f..529a2248b3 100644 --- a/cmd/createcluster_internal_test.go +++ b/cmd/createcluster_internal_test.go @@ -1250,13 +1250,13 @@ func TestZipped(t *testing.T) { require.NoError(t, app.BundleOutput(conf.ClusterDir, "cluster.tar.gz")) unzippedDir := t.TempDir() - require.NoError(t, app.ExtractArchive(filepath.Join(conf.ClusterDir, "cluster.tar.gz"), unzippedDir)) + require.NoError(t, testutil.ExtractArchive(filepath.Join(conf.ClusterDir, "cluster.tar.gz"), unzippedDir)) err := os.RemoveAll(filepath.Join(conf.ClusterDir, "cluster.tar.gz")) require.NoError(t, err) // Compare directories using the shared function - err = app.CompareDirectories(backupDir, unzippedDir) + err = testutil.CompareDirectories(backupDir, unzippedDir) require.NoError(t, err, "unzipped directory should match original directory structure") } diff --git a/dkg/dkg_test.go b/dkg/dkg_test.go index 8014951d02..0615f2852c 100644 --- a/dkg/dkg_test.go +++ b/dkg/dkg_test.go @@ -843,11 +843,11 @@ func TestZipped(t *testing.T) { // Create new temp dir and unzip to there unzippedDir := t.TempDir() - err = app.ExtractArchive(tarPath, unzippedDir) + err = testutil.ExtractArchive(tarPath, unzippedDir) require.NoError(t, err) // Compare backup with unzipped folder - err = app.CompareDirectories(backupDir, unzippedDir) + err = testutil.CompareDirectories(backupDir, unzippedDir) require.NoError(t, err, "unzipped content should match original for node0") } diff --git a/testutil/archive.go b/testutil/archive.go new file mode 100644 index 0000000000..3c2a0acab5 --- /dev/null +++ b/testutil/archive.go @@ -0,0 +1,186 @@ +// Copyright © 2022-2026 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package testutil + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + + "github.com/obolnetwork/charon/app/errors" + "github.com/obolnetwork/charon/app/z" +) + +// ExtractArchive extracts a .tar.gz archive to the target directory. +func ExtractArchive(archivePath, targetDir string) error { + file, err := os.Open(archivePath) + if err != nil { + return errors.Wrap(err, "open archive file") + } + defer file.Close() + + gzr, err := gzip.NewReader(file) + if err != nil { + return errors.Wrap(err, "create gzip reader") + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil // End of archive + } + + if err != nil { + return errors.Wrap(err, "tar read error") + } + + cleanName := filepath.Clean(header.Name) + // Disallow absolute paths and paths with ".." as first element + if strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) || strings.Contains(cleanName, ".."+string(os.PathSeparator)) { + return errors.New("invalid archive entry path", z.Str("path", header.Name)) + } + + target := filepath.Join(targetDir, cleanName) + + absTargetDir, err := filepath.Abs(targetDir) + if err != nil { + return errors.Wrap(err, "get absolute target dir") + } + + absTarget, err := filepath.Abs(target) + if err != nil { + return errors.Wrap(err, "get absolute target path") + } + + if !strings.HasPrefix(absTarget, absTargetDir+string(os.PathSeparator)) && absTarget != absTargetDir { + return errors.New("archive entry path escapes target directory", z.Str("path", header.Name)) + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, header.FileInfo().Mode()); err != nil { + return errors.Wrap(err, "create directory") + } + case tar.TypeReg: + // Check file size to prevent decompression bombs + const maxFileSize = 100 * 1024 * 1024 // 100MB limit per file + if header.Size > maxFileSize { + return errors.New("file too large in archive", z.Str("path", header.Name), z.I64("size", header.Size)) + } + + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return errors.Wrap(err, "create parent directory") + } + + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, header.FileInfo().Mode()) + if err != nil { + return errors.Wrap(err, "create file") + } + + // Use limited reader to prevent decompression bombs + limitedReader := io.LimitReader(tr, maxFileSize) + + _, err = io.Copy(f, limitedReader) + if err != nil { + f.Close() + return errors.Wrap(err, "copy file contents") + } + + if err := f.Close(); err != nil { + return errors.Wrap(err, "close file") + } + default: + // Skip other types (symlinks, etc.) + } + } +} + +// CompareDirectories recursively compares two directories and their contents. +func CompareDirectories(originalDir, extractedDir string) error { + err := filepath.Walk(originalDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(originalDir, path) + if err != nil { + return errors.Wrap(err, "get relative path") + } + + extractedPath := filepath.Join(extractedDir, relPath) + + if info.IsDir() { + // Check if directory exists in extracted content + extractedInfo, err := os.Stat(extractedPath) + if err != nil { + return errors.Wrap(err, "directory should exist in extracted content", z.Str("path", relPath)) + } + + if !extractedInfo.IsDir() { + return errors.New("path is not a directory", z.Str("path", relPath)) + } + + return nil + } + + // Compare file contents + extractedInfo, err := os.Stat(extractedPath) + if err != nil { + return errors.Wrap(err, "file should exist in extracted content", z.Str("path", relPath)) + } + + if info.Size() != extractedInfo.Size() { + return errors.New("file size mismatch", z.Str("path", relPath)) + } + + // Read and compare file contents + originalContent, err := os.ReadFile(path) //nolint:gosec // no TOCTOU condition as CompareDirectories is used only in tests. + if err != nil { + return errors.Wrap(err, "read original file", z.Str("path", path)) + } + + extractedContent, err := os.ReadFile(extractedPath) + if err != nil { + return errors.Wrap(err, "read extracted file", z.Str("path", extractedPath)) + } + + if !bytes.Equal(originalContent, extractedContent) { + return errors.New("file content mismatch", z.Str("path", relPath)) + } + + return nil + }) + if err != nil { + return errors.Wrap(err, "walk directory tree") + } + + // Check that the extracted content contains nothing else. + err = filepath.Walk(extractedDir, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(extractedDir, path) + if err != nil { + return errors.Wrap(err, "get relative path") + } + + if _, err := os.Stat(filepath.Join(originalDir, relPath)); err != nil { + return errors.Wrap(err, "path should not exist in extracted content", z.Str("path", relPath)) + } + + return nil + }) + if err != nil { + return errors.Wrap(err, "walk extracted directory tree") + } + + return nil +} diff --git a/testutil/archive_test.go b/testutil/archive_test.go new file mode 100644 index 0000000000..e34fde267b --- /dev/null +++ b/testutil/archive_test.go @@ -0,0 +1,213 @@ +// Copyright © 2022-2026 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package testutil_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/obolnetwork/charon/testutil" +) + +func TestCompareDirectories(t *testing.T) { + tests := []struct { + name string + setupDirs func(t *testing.T) (dir1, dir2 string) + expectError bool + errorMsg string + }{ + { + name: "identical_directories", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + testFiles := map[string]string{ + "file1.txt": "content1", + "nested/file2.json": `{"key": "value"}`, + "nested/deep/file3.md": "# Header\nContent", + "binary.bin": "\x00\x01\x02\x03", + "special_chars_äöü.txt": "Special characters: äöüß", + } + + for relPath, content := range testFiles { + fullPath := filepath.Join(dir1, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0o644)) + } + + dir2 := t.TempDir() + require.NoError(t, os.CopyFS(dir2, os.DirFS(dir1))) + + return dir1, dir2 + }, + expectError: false, + }, + { + name: "missing_file", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir1, "file.txt"), []byte("content"), 0o644)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "file should exist in extracted content", + }, + { + name: "extra_file", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir2, "file.txt"), []byte("content"), 0o644)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "path should not exist in extracted content", + }, + { + name: "extra_directory", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir2, "sub"), 0o755)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "path should not exist in extracted content", + }, + { + name: "different_content", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir1, "file.txt"), []byte("content1"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir2, "file.txt"), []byte("content2"), 0o644)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "file content mismatch", + }, + { + name: "different_sizes", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir1, "file.txt"), []byte("short"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir2, "file.txt"), []byte("much longer content"), 0o644)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "file size mismatch", + }, + { + name: "missing_directory", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + nestedDir := filepath.Join(dir1, "nested", "deep") + require.NoError(t, os.MkdirAll(nestedDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nestedDir, "file.txt"), []byte("content"), 0o644)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "directory should exist in extracted content", + }, + { + name: "file_vs_directory", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir1, "item"), []byte("content"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir2, "item"), 0o755)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "file size mismatch", + }, + { + name: "directory_vs_file", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir1, "item"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir2, "item"), []byte("content"), 0o644)) + + return dir1, dir2 + }, + expectError: true, + errorMsg: "path is not a directory", + }, + { + name: "complex_structure", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + dir1 := t.TempDir() + dir2 := t.TempDir() + files := map[string]string{ + "root.txt": "root content", + "validator_keys/keystore-1.json": `{"crypto": {"cipher": "test"}}`, + "validator_keys/keystore-2.json": `{"crypto": {"cipher": "test"}}`, + "nested/level1/level2/deep.yaml": "key: value\narray:\n - item1\n - item2", + "cluster-lock.json": `{"lock_hash": "0xabc"}`, + "deposit_data.json": `[{"pubkey": "0x123"}]`, + "empty_dir/placeholder.txt": "", + "binary_data.bin": "\x00\x01\x02\x03\xFF\xFE\xFD", + } + + for relPath, content := range files { + for _, dir := range []string{dir1, dir2} { + fullPath := filepath.Join(dir, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0o755)) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0o644)) + } + } + + return dir1, dir2 + }, + expectError: false, + }, + { + name: "empty_directories", + setupDirs: func(t *testing.T) (string, string) { + t.Helper() + return t.TempDir(), t.TempDir() + }, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir1, dir2 := tt.setupDirs(t) + err := testutil.CompareDirectories(dir1, dir2) + + if tt.expectError { + require.Error(t, err, "Expected error for test case: %s", tt.name) + + if tt.errorMsg != "" { + require.Contains(t, err.Error(), tt.errorMsg, "Error message should contain expected text") + } + } else { + require.NoError(t, err, "Expected no error for test case: %s", tt.name) + } + }) + } +}