diff --git a/cmd/backup/config.go b/cmd/backup/config.go index e68a2c99..d2383cf4 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -82,6 +82,8 @@ type Config struct { SSHIdentityPassphrase string `split_words:"true"` SSHRemotePath string `split_words:"true"` SSHInsecureIgnoreHostKey bool `split_words:"true"` + SSHUploadRetries WholeNumber `split_words:"true" default:"0"` + SSHUploadRetryDelay time.Duration `split_words:"true" default:"5s"` ExecLabel string `split_words:"true"` ExecForwardOutput bool `split_words:"true"` LockTimeout time.Duration `split_words:"true" default:"60m"` diff --git a/cmd/backup/config_ssh_test.go b/cmd/backup/config_ssh_test.go new file mode 100644 index 00000000..49b610ec --- /dev/null +++ b/cmd/backup/config_ssh_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 - offen.software +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/offen/envconfig" +) + +func TestSSHUploadRetryConfiguration(t *testing.T) { + originalLookup := envconfig.Lookup + t.Cleanup(func() { envconfig.Lookup = originalLookup }) + for _, test := range []struct { + name string + values map[string]string + retries int + delay time.Duration + invalid bool + }{ + {name: "defaults", retries: 0, delay: 5 * time.Second}, + {name: "disabled", values: map[string]string{"SSH_UPLOAD_RETRIES": "0"}, delay: 5 * time.Second}, + {name: "custom", values: map[string]string{"SSH_UPLOAD_RETRIES": "4", "SSH_UPLOAD_RETRY_DELAY": "500ms"}, retries: 4, delay: 500 * time.Millisecond}, + {name: "negative retries", values: map[string]string{"SSH_UPLOAD_RETRIES": "-1"}, invalid: true}, + {name: "invalid delay", values: map[string]string{"SSH_UPLOAD_RETRY_DELAY": "invalid"}, invalid: true}, + } { + t.Run(test.name, func(t *testing.T) { + config, err := loadConfig(func(key string) (string, bool) { + value, ok := test.values[key] + return value, ok + }) + if test.invalid { + if err == nil { + t.Fatal("Expected invalid retry configuration to be rejected") + } + return + } + if err != nil { + t.Fatal(err) + } + if int(config.SSHUploadRetries) != test.retries || config.SSHUploadRetryDelay != test.delay { + t.Errorf("Got retries=%d, delay=%s; want %d, %s", config.SSHUploadRetries, config.SSHUploadRetryDelay, test.retries, test.delay) + } + }) + } +} diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 5a0b0841..93c7f894 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -196,6 +196,8 @@ func (s *script) init() error { InsecureIgnoreHostKey: s.c.SSHInsecureIgnoreHostKey, HostPublicKey: s.c.SSHHostPublicKey, KnownHostsFile: s.c.SSHKnownHostsFile, + UploadRetries: int(s.c.SSHUploadRetries), + UploadRetryDelay: s.c.SSHUploadRetryDelay, } sshBackend, closeSSHConnection, err := ssh.NewStorageBackend(sshConfig, logFunc) diff --git a/docs/reference/index.md b/docs/reference/index.md index 406cd8f1..83506053 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -361,6 +361,25 @@ If you need to confirm what the container actually loaded, see [Show loaded conf # SSH_IGNORE_HOST_KEY="false" +# --- + +# The number of additional upload attempts after an SSH/SFTP connection error. +# Each retry opens a new connection and transfers the existing archive from the +# beginning; containers are not stopped again and the archive is not recreated. +# Permission errors and other non-connection errors are not retried. +# Retries are disabled by default. Set to a positive number to enable them. +# Initial connection failures during backup setup and pruning are not retried. +# Exhausted retries still fail the backup run; this setting does not retain +# temporary archives after a failed run. + +# SSH_UPLOAD_RETRIES="0" + +# --- + +# The delay before each upload retry, as a non-negative duration. + +# SSH_UPLOAD_RETRY_DELAY="5s" + ########### AZURE BLOB STORAGE # The credential's account name when using Azure Blob Storage. This has to be diff --git a/internal/storage/ssh/retry_test.go b/internal/storage/ssh/retry_test.go new file mode 100644 index 00000000..cdc37f14 --- /dev/null +++ b/internal/storage/ssh/retry_test.go @@ -0,0 +1,336 @@ +// Copyright 2026 - offen.software +// SPDX-License-Identifier: MPL-2.0 + +package ssh + +import ( + "bytes" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/offen/docker-volume-backup/internal/storage" + "github.com/pkg/sftp" +) + +type uploadTestClient struct { + mkdirErr error + create func(string) (io.WriteCloser, error) +} + +func (c *uploadTestClient) MkdirAll(string) error { return c.mkdirErr } +func (c *uploadTestClient) Create(name string) (io.WriteCloser, error) { + return c.create(name) +} +func (c *uploadTestClient) ReadDir(string) ([]os.FileInfo, error) { return nil, nil } +func (c *uploadTestClient) Remove(string) error { return nil } + +type uploadTestFile struct { + write func([]byte) (int, error) + close func() error +} + +func (f *uploadTestFile) Write(p []byte) (int, error) { return f.write(p) } +func (f *uploadTestFile) Close() error { return f.close() } + +func uploadTestSource(t *testing.T) (string, []byte) { + t.Helper() + data := bytes.Repeat([]byte("a completed, encrypted backup\n"), 4096) + name := filepath.Join(t.TempDir(), "backup.tar.gz.age") + if err := os.WriteFile(name, data, 0o600); err != nil { + t.Fatal(err) + } + return name, data +} + +func uploadTestBackend(client sftpClient, retries int, logs *[]string) *sshStorage { + return &sshStorage{ + StorageBackend: &storage.StorageBackend{ + DestinationPath: "/backups", + Log: func(level storage.LogLevel, _ string, message string, args ...any) { + if level == storage.LogLevelInfo { + *logs = append(*logs, fmt.Sprintf(message, args...)) + } + }, + }, + sftpClient: client, + closeSession: noop, + uploadRetries: retries, + } +} + +func TestCopyReconnectsAndRestartsUpload(t *testing.T) { + file, data := uploadTestSource(t) + synctest.Test(t, func(t *testing.T) { + var remote bytes.Buffer + var logs []string + var connections, creates, oldCloses, newCloses, fileCloses int + first := &uploadTestClient{create: func(name string) (io.WriteCloser, error) { + if name != "/backups/backup.tar.gz.age" { + t.Fatalf("Destination = %q", name) + } + creates++ + remote.Reset() + return &uploadTestFile{ + write: func(p []byte) (int, error) { + n, _ := remote.Write(p[:13]) + return n, sftp.ErrSSHFxConnectionLost + }, + close: func() error { fileCloses++; return io.EOF }, + }, nil + }} + b := uploadTestBackend(first, 2, &logs) + b.uploadRetryDelay = 5 * time.Second + b.closeSession = func() error { + oldCloses++ + return errors.New("old transport already closed") + } + started := time.Now() + b.connect = func() (sftpClient, func() error, error) { + connections++ + if oldCloses != 1 || fileCloses != 1 { + t.Fatalf("Reconnect before closing old session/file: %d/%d", oldCloses, fileCloses) + } + if elapsed := time.Since(started); elapsed != b.uploadRetryDelay { + t.Fatalf("Retry after %s, want %s", elapsed, b.uploadRetryDelay) + } + client := &uploadTestClient{create: func(string) (io.WriteCloser, error) { + creates++ + remote.Reset() + return &uploadTestFile{ + write: remote.Write, + close: func() error { + if len(logs) != 0 { + t.Error("Success logged before closing the remote file") + } + fileCloses++ + return nil + }, + }, nil + }} + return client, func() error { newCloses++; return nil }, nil + } + if err := b.Copy(file); err != nil { + t.Fatalf("Copy: %v", err) + } + if !bytes.Equal(remote.Bytes(), data) { + t.Fatal("Retry did not upload the complete source from byte zero") + } + if connections != 1 || creates != 2 || fileCloses != 2 || len(logs) != 1 { + t.Fatalf("connections=%d, creates=%d, fileCloses=%d, successLogs=%d", connections, creates, fileCloses, len(logs)) + } + if err := b.close(); err != nil { + t.Fatalf("Cleanup retained the old session's close error: %v", err) + } + if err := b.close(); err != nil || newCloses != 1 { + t.Fatalf("Repeated cleanup: err=%v, newCloses=%d", err, newCloses) + } + }) +} + +func TestCopyRetriesRemoteCloseFailure(t *testing.T) { + file, data := uploadTestSource(t) + var logs []string + var uploads int + client := func(closeErr error) *uploadTestClient { + return &uploadTestClient{create: func(string) (io.WriteCloser, error) { + uploads++ + var received bytes.Buffer + return &uploadTestFile{ + write: received.Write, + close: func() error { + if !bytes.Equal(received.Bytes(), data) { + t.Error("Upload differs from source") + } + if len(logs) != 0 { + t.Error("Success logged before remote close") + } + return closeErr + }, + }, nil + }} + } + b := uploadTestBackend(client(io.EOF), 1, &logs) + b.connect = func() (sftpClient, func() error, error) { return client(nil), noop, nil } + if err := b.Copy(file); err != nil { + t.Fatal(err) + } + if uploads != 2 || len(logs) != 1 { + t.Fatalf("uploads=%d, successLogs=%d; want 2, 1", uploads, len(logs)) + } +} + +func TestCopyRetryLimit(t *testing.T) { + file, _ := uploadTestSource(t) + for _, reconnectFails := range []bool{false, true} { + t.Run(fmt.Sprintf("reconnectFails=%v", reconnectFails), func(t *testing.T) { + var logs []string + var connections, closes int + client := &uploadTestClient{mkdirErr: sftp.ErrSSHFxConnectionLost} + b := uploadTestBackend(client, 2, &logs) + b.closeSession = func() error { closes++; return nil } + b.connect = func() (sftpClient, func() error, error) { + connections++ + if reconnectFails { + return nil, noop, io.ErrUnexpectedEOF + } + return client, func() error { closes++; return nil }, nil + } + if err := b.Copy(file); err == nil { + t.Fatal("Copy succeeded after exhausting retries") + } + if connections != 2 || len(logs) != 0 { + t.Fatalf("connections=%d, successLogs=%d; want 2, 0", connections, len(logs)) + } + if err := b.close(); err != nil { + t.Fatal(err) + } + wantCloses := 3 + if reconnectFails { + wantCloses = 1 + } + if closes != wantCloses { + t.Fatalf("Closed %d sessions, want %d", closes, wantCloses) + } + }) + } +} + +func TestCopyDoesNotRetryPermanentErrorsOrWhenDisabled(t *testing.T) { + file, _ := uploadTestSource(t) + for _, tt := range []struct { + name string + err error + retries int + }{ + {"disabled", sftp.ErrSSHFxConnectionLost, 0}, + {"permission", os.ErrPermission, 2}, + {"remote failure", sftp.ErrSSHFxFailure, 2}, + } { + t.Run(tt.name, func(t *testing.T) { + var logs []string + b := uploadTestBackend(&uploadTestClient{mkdirErr: tt.err}, tt.retries, &logs) + b.connect = func() (sftpClient, func() error, error) { + t.Fatal("Unexpected retry") + return nil, noop, nil + } + if err := b.Copy(file); !errors.Is(err, tt.err) { + t.Fatalf("Copy error = %v, want %v", err, tt.err) + } + if len(logs) != 0 { + t.Fatal("Success logged for a failed upload") + } + }) + } +} + +func TestCopyDoesNotRetryMissingSource(t *testing.T) { + var logs []string + b := uploadTestBackend(nil, 2, &logs) + if err := b.Copy(filepath.Join(t.TempDir(), "missing")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("Missing source: %v", err) + } +} + +func TestCopyDoesNotRetryPermanentWriteErrorWithFailedClose(t *testing.T) { + file, _ := uploadTestSource(t) + var logs []string + var closes int + client := &uploadTestClient{create: func(string) (io.WriteCloser, error) { + return &uploadTestFile{ + write: func([]byte) (int, error) { return 0, os.ErrPermission }, + close: func() error { closes++; return io.EOF }, + }, nil + }} + b := uploadTestBackend(client, 2, &logs) + b.connect = func() (sftpClient, func() error, error) { + t.Fatal("A close error caused a permanent write error to be retried") + return nil, noop, nil + } + err := b.Copy(file) + if !errors.Is(err, os.ErrPermission) || !errors.Is(err, io.EOF) { + t.Fatalf("Copy error = %v, want permission and close errors", err) + } + if closes != 1 || len(logs) != 0 { + t.Fatalf("closes=%d, successLogs=%d; want 1, 0", closes, len(logs)) + } +} + +func TestCopyFileChecksSizeAndClosesRemoteFile(t *testing.T) { + file, data := uploadTestSource(t) + source, err := os.Open(file) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Errorf("Close source: %v", err) + } + }) + var logs []string + var closes int + client := &uploadTestClient{create: func(string) (io.WriteCloser, error) { + return &uploadTestFile{ + write: io.Discard.Write, + close: func() error { closes++; return nil }, + }, nil + }} + b := uploadTestBackend(client, 2, &logs) + err = b.copyFile(source, int64(len(data))+1, filepath.Base(file)) + if err == nil || !strings.Contains(err.Error(), "failed to upload the file completely") { + t.Fatalf("Size mismatch error = %v", err) + } + if retryableUploadError(err) || closes != 1 { + t.Fatalf("Size mismatch retryable=%v, closes=%d", retryableUploadError(err), closes) + } +} + +func TestRetryableUploadError(t *testing.T) { + for _, tt := range []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"EOF", io.EOF, true}, + {"unexpected EOF", io.ErrUnexpectedEOF, true}, + {"closed pipe", io.ErrClosedPipe, true}, + {"closed network", net.ErrClosed, true}, + {"network operation", &net.OpError{Op: "write", Net: "tcp", Err: errors.New("reset")}, true}, + {"SFTP lost", sftp.ErrSSHFxConnectionLost, true}, + {"SFTP disconnected", sftp.ErrSSHFxNoConnection, true}, + {"SFTP status lost", &sftp.StatusError{Code: uint32(sftp.ErrSSHFxConnectionLost)}, true}, + {"SFTP status permission", &sftp.StatusError{Code: uint32(sftp.ErrSSHFxPermissionDenied)}, false}, + {"SFTP no space", &sftp.StatusError{Code: 14}, false}, + {"SFTP failure", sftp.ErrSSHFxFailure, false}, + {"permission", os.ErrPermission, false}, + {"local read", &os.PathError{Op: "read", Path: "backup", Err: io.ErrUnexpectedEOF}, false}, + {"unrecognized", errors.New("connection lost"), false}, + {"joined transport", errors.Join(io.EOF, sftp.ErrSSHFxConnectionLost), true}, + {"joined permanent", errors.Join(os.ErrPermission, io.EOF), false}, + {"wrapped joined permanent", fmt.Errorf("upload: %w", errors.Join(os.ErrPermission, io.EOF)), false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := retryableUploadError(tt.err); got != tt.want { + t.Fatalf("retryableUploadError(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} + +func TestNewStorageBackendRejectsNegativeRetryDelay(t *testing.T) { + _, cleanup, err := NewStorageBackend(Config{UploadRetryDelay: -time.Second}, nil) + if err == nil || !strings.Contains(err.Error(), "must not be negative") { + t.Fatalf("Expected negative retry delay to be rejected, got: %v", err) + } + if err := cleanup(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/storage/ssh/ssh.go b/internal/storage/ssh/ssh.go index e647bf91..6b120b69 100644 --- a/internal/storage/ssh/ssh.go +++ b/internal/storage/ssh/ssh.go @@ -7,9 +7,12 @@ import ( "errors" "fmt" "io" + "net" "os" "path" + "path/filepath" "strings" + "sync" "time" "github.com/offen/docker-volume-backup/internal/errwrap" @@ -21,9 +24,25 @@ import ( type sshStorage struct { *storage.StorageBackend - client *ssh.Client - sftpClient *sftp.Client - hostName string + sftpClient sftpClient + connect func() (sftpClient, func() error, error) + closeSession func() error + hostName string + uploadRetries int + uploadRetryDelay time.Duration +} + +type sftpClient interface { + MkdirAll(string) error + Create(string) (io.WriteCloser, error) + ReadDir(string) ([]os.FileInfo, error) + Remove(string) error +} + +type sftpConnection struct{ *sftp.Client } + +func (c sftpConnection) Create(name string) (io.WriteCloser, error) { + return c.Client.Create(name) } // Config allows to configure a SSH backend. @@ -38,12 +57,17 @@ type Config struct { RemotePath string KnownHostsFile string InsecureIgnoreHostKey bool + UploadRetries int + UploadRetryDelay time.Duration } var noop = func() error { return nil } // NewStorageBackend creates and initializes a new SSH storage backend. func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, func() error, error) { + if opts.UploadRetryDelay < 0 { + return nil, noop, errwrap.Wrap(nil, "SSH upload retry delay must not be negative") + } var authMethods []ssh.AuthMethod if opts.Password != "" { @@ -93,33 +117,54 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, func( sshClientConfig.HostKeyCallback = cb } - sshClient, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", opts.HostName, opts.Port), sshClientConfig) - if err != nil || sshClient == nil { - return nil, noop, errwrap.Wrap(err, "error creating ssh client") - } - _, _, err = sshClient.SendRequest("keepalive", false, nil) - if err != nil { - return nil, sshClient.Close, err - } + connect := func() (sftpClient, func() error, error) { + client, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", opts.HostName, opts.Port), sshClientConfig) + if err != nil { + return nil, noop, errwrap.Wrap(err, "error creating ssh client") + } + if _, _, err := client.SendRequest("keepalive", false, nil); err != nil { + _ = client.Close() + return nil, noop, err + } - sftpClient, err := sftp.NewClient(sshClient, - sftp.UseConcurrentReads(true), - sftp.UseConcurrentWrites(true), - sftp.MaxConcurrentRequestsPerFile(64), - ) - if err != nil { - return nil, sshClient.Close, errwrap.Wrap(err, "error creating sftp client") + sftpClient, err := sftp.NewClient(client, + sftp.UseConcurrentReads(true), + sftp.UseConcurrentWrites(true), + sftp.MaxConcurrentRequestsPerFile(64), + ) + if err != nil { + _ = client.Close() + return nil, noop, errwrap.Wrap(err, "error creating sftp client") + } + return sftpConnection{sftpClient}, startKeepAlive(client, keepAliveInterval), nil } - return &sshStorage{ + b := &sshStorage{ StorageBackend: &storage.StorageBackend{ DestinationPath: opts.RemotePath, Log: logFunc, }, - client: sshClient, - sftpClient: sftpClient, - hostName: opts.HostName, - }, startKeepAlive(sshClient, keepAliveInterval), nil + connect: connect, + hostName: opts.HostName, + uploadRetries: opts.UploadRetries, + uploadRetryDelay: opts.UploadRetryDelay, + } + var err error + b.sftpClient, b.closeSession, err = connect() + if err != nil { + return nil, noop, err + } + // Resolve the current session at cleanup time, as Copy can replace it. + return b, sync.OnceValue(b.close), nil +} + +func (b *sshStorage) close() error { + if b.closeSession == nil { + return nil + } + closeSession := b.closeSession + b.closeSession = nil + return closeSession() } // Name returns the name of the storage backend @@ -129,15 +174,9 @@ func (b *sshStorage) Name() string { // Copy copies the given file to the SSH storage backend. func (b *sshStorage) Copy(file string) (returnErr error) { - if err := b.sftpClient.MkdirAll(b.DestinationPath); err != nil { - returnErr = errwrap.Wrap(err, "error ensuring destination directory") - return - } - source, err := os.Open(file) - _, name := path.Split(file) if err != nil { - returnErr = errwrap.Wrap(err, " error reading the file to be uploaded") + returnErr = errwrap.Wrap(err, "error reading the file to be uploaded") return } defer func() { @@ -150,13 +189,44 @@ func (b *sshStorage) Copy(file string) (returnErr error) { return } + err = b.copyFile(source, sourceFileInfo.Size(), filepath.Base(file)) + for retry := 0; err != nil && retry < b.uploadRetries && retryableUploadError(err); retry++ { + // The failed session (including its keepalive) must be stopped before + // reconnecting. Its close error must not taint a successful later upload. + _ = b.close() + b.Log(storage.LogLevelWarning, b.Name(), "SSH upload failed, retrying (%d/%d) in %s: %v", retry+1, b.uploadRetries, b.uploadRetryDelay, err) + time.Sleep(b.uploadRetryDelay) + b.sftpClient, b.closeSession, err = b.connect() + if err != nil { + continue + } + err = b.copyFile(source, sourceFileInfo.Size(), filepath.Base(file)) + } + if err != nil { + return err + } + + b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to '%s' at path '%s'.", file, b.hostName, b.DestinationPath) + return nil +} + +func (b *sshStorage) copyFile(source *os.File, size int64, name string) (returnErr error) { + if _, err := source.Seek(0, io.SeekStart); err != nil { + return errwrap.Wrap(err, "error rewinding the source file") + } + if err := b.sftpClient.MkdirAll(b.DestinationPath); err != nil { + return errwrap.Wrap(err, "error ensuring destination directory") + } + // Create truncates a partial upload left by an earlier attempt. destination, err := b.sftpClient.Create(path.Join(b.DestinationPath, name)) if err != nil { returnErr = errwrap.Wrap(err, "error creating file") return } defer func() { - returnErr = errors.Join(returnErr, destination.Close()) + if err := destination.Close(); err != nil { + returnErr = errors.Join(returnErr, errwrap.Wrap(err, "error closing the uploaded file")) + } }() written, err := io.Copy(destination, source) @@ -165,22 +235,53 @@ func (b *sshStorage) Copy(file string) (returnErr error) { return } - if written != sourceFileInfo.Size() { + if written != size { msg := fmt.Sprintf( "failed to upload the file completely: wrote %d, expected %d", written, - sourceFileInfo.Size(), + size, ) returnErr = errwrap.Wrap(err, msg) return } - b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to '%s' at path '%s'.", file, b.hostName, b.DestinationPath) - return nil } +// Retry only transport failures. A joined permanent error, such as a full disk +// followed by a failed close, must not become retryable because of the close. +func retryableUploadError(err error) bool { + if err == nil { + return false + } + switch err := err.(type) { + case interface{ Unwrap() []error }: + causes := err.Unwrap() + if len(causes) == 0 { + return false + } + for _, cause := range causes { + if !retryableUploadError(cause) { + return false + } + } + return true + case *os.PathError: + return false // Local source open, read, stat, and seek errors. + case *sftp.StatusError: + return err.FxCode() == sftp.ErrSSHFxConnectionLost || err.FxCode() == sftp.ErrSSHFxNoConnection + case net.Error: + return true + } + switch err { + case io.EOF, io.ErrUnexpectedEOF, io.ErrClosedPipe, net.ErrClosed, + sftp.ErrSSHFxConnectionLost, sftp.ErrSSHFxNoConnection: + return true + } + return retryableUploadError(errors.Unwrap(err)) +} + // Prune rotates away backups according to the configuration and provided deadline for the SSH storage backend. func (b *sshStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { candidates, err := b.sftpClient.ReadDir(b.DestinationPath) diff --git a/internal/storage/ssh/ssh_integration_test.go b/internal/storage/ssh/ssh_integration_test.go new file mode 100644 index 00000000..78a7b977 --- /dev/null +++ b/internal/storage/ssh/ssh_integration_test.go @@ -0,0 +1,212 @@ +// Copyright 2026 - offen.software +// SPDX-License-Identifier: MPL-2.0 + +package ssh + +import ( + "bytes" + "crypto/ed25519" + "crypto/rand" + "io" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/offen/docker-volume-backup/internal/storage" + "github.com/pkg/sftp" + gossh "golang.org/x/crypto/ssh" +) + +// Exercise the real SSH/SFTP clients: the first connection is severed after +// receiving part of the archive, then the retry must upload every byte again. +func TestCopyReconnectsAfterInterruptedSFTPUpload(t *testing.T) { + handlers := sftp.InMemHandler() + address, hostKey, accepted, disconnected := startInterruptingSFTPServer(t, handlers) + host, port, err := net.SplitHostPort(address) + if err != nil { + t.Fatal(err) + } + contents := bytes.Repeat([]byte("encrypted-backup-content\n"), 32768) + file := filepath.Join(t.TempDir(), "backup.tar.gz.age") + if err := os.WriteFile(file, contents, 0600); err != nil { + t.Fatal(err) + } + var successes, warnings int + backend, cleanup, err := NewStorageBackend(Config{ + HostName: host, Port: port, User: "test", RemotePath: "/backups", + HostPublicKey: string(gossh.MarshalAuthorizedKey(hostKey)), + UploadRetries: 2, UploadRetryDelay: time.Millisecond, + }, func(level storage.LogLevel, _ string, message string, _ ...any) { + if level == storage.LogLevelInfo && strings.HasPrefix(message, "Uploaded a copy") { + successes++ + } + if level == storage.LogLevelWarning { + warnings++ + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cleanup() }) + // Production uses Unix paths; normalize the temporary Windows path as well. + if err := backend.Copy(filepath.ToSlash(file)); err != nil { + t.Fatalf("Copy did not recover from the interrupted upload: %v", err) + } + if accepted.Load() != 2 || disconnected.Load() != 1 { + t.Fatalf("connections=%d, injected disconnects=%d; want 2, 1", accepted.Load(), disconnected.Load()) + } + if successes != 1 || warnings != 1 { + t.Fatalf("success logs=%d, retry warnings=%d; want 1 each", successes, warnings) + } + request := sftp.NewRequest("Get", "/backups/backup.tar.gz.age") + request.Flags = 1 // SSH_FXF_READ + remote, err := handlers.FileGet.Fileread(request) + if err != nil { + t.Fatal(err) + } + if closer, ok := remote.(io.Closer); ok { + defer func() { _ = closer.Close() }() + } + actual, err := io.ReadAll(io.NewSectionReader(remote, 0, int64(len(contents)+1))) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(actual, contents) { + t.Fatalf("remote archive differs from source: got %d bytes, want %d", len(actual), len(contents)) + } + // Pruning must also use the replacement session, not the failed one. + if _, err := backend.Prune(time.Now().Add(-time.Hour), "backup-"); err != nil { + t.Fatalf("Prune after reconnect: %v", err) + } + if err := cleanup(); err != nil { + t.Fatalf("Cleanup reported the discarded connection's error: %v", err) + } +} + +type interruptedSFTPWriter struct { + sftp.FileWriter + connection net.Conn + disconnected *atomic.Int32 +} + +func (w *interruptedSFTPWriter) OpenFile(request *sftp.Request) (sftp.WriterAtReaderAt, error) { + file, err := w.FileWriter.(sftp.OpenFileWriter).OpenFile(request) + if err != nil { + return nil, err + } + return &interruptedSFTPFile{WriterAtReaderAt: file, writer: w}, nil +} + +type interruptedSFTPFile struct { + sftp.WriterAtReaderAt + writer *interruptedSFTPWriter +} + +func (f *interruptedSFTPFile) WriteAt(p []byte, offset int64) (int, error) { + if offset >= 32*1024 { + if f.writer.disconnected.CompareAndSwap(0, 1) { + _ = f.writer.connection.Close() + } + return 0, io.ErrClosedPipe + } + return f.WriterAtReaderAt.WriteAt(p, offset) +} + +func (f *interruptedSFTPFile) Close() error { + if closer, ok := f.WriterAtReaderAt.(io.Closer); ok { + return closer.Close() + } + return nil +} + +func startInterruptingSFTPServer(t *testing.T, handlers sftp.Handlers) (string, gossh.PublicKey, *atomic.Int32, *atomic.Int32) { + t.Helper() + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + signer, err := gossh.NewSignerFromKey(privateKey) + if err != nil { + t.Fatal(err) + } + config := &gossh.ServerConfig{NoClientAuth: true} + config.AddHostKey(signer) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var accepted, disconnected atomic.Int32 + var connections []net.Conn + var mu sync.Mutex + var workers sync.WaitGroup + acceptDone := make(chan struct{}) + t.Cleanup(func() { + _ = listener.Close() + <-acceptDone + mu.Lock() + for _, connection := range connections { + _ = connection.Close() + } + mu.Unlock() + workers.Wait() + }) + go func() { + defer close(acceptDone) + for { + connection, err := listener.Accept() + if err != nil { + return + } + mu.Lock() + connections = append(connections, connection) + mu.Unlock() + number := accepted.Add(1) + workers.Add(1) + go func() { + defer workers.Done() + defer func() { _ = connection.Close() }() + _ = connection.SetDeadline(time.Now().Add(15 * time.Second)) + server, channels, requests, err := gossh.NewServerConn(connection, config) + if err != nil { + return + } + defer func() { _ = server.Close() }() + workers.Go(func() { gossh.DiscardRequests(requests) }) + for incoming := range channels { + if incoming.ChannelType() != "session" { + _ = incoming.Reject(gossh.UnknownChannelType, "expected session") + continue + } + channel, requests, err := incoming.Accept() + if err != nil { + return + } + workers.Go(func() { + defer func() { _ = channel.Close() }() + for request := range requests { + var subsystem struct{ Name string } + valid := request.Type == "subsystem" && gossh.Unmarshal(request.Payload, &subsystem) == nil && subsystem.Name == "sftp" + if err := request.Reply(valid, nil); err != nil || !valid { + continue + } + sessionHandlers := handlers + if number == 1 { + sessionHandlers.FilePut = &interruptedSFTPWriter{FileWriter: handlers.FilePut, connection: connection, disconnected: &disconnected} + } + sftpServer := sftp.NewRequestServer(channel, sessionHandlers) + _ = sftpServer.Serve() + _ = sftpServer.Close() + return + } + }) + } + }() + } + }() + return listener.Addr().String(), signer.PublicKey(), &accepted, &disconnected +}