From e32fc72583b8f5c90f8c6b7c56d84f926eb96dc0 Mon Sep 17 00:00:00 2001 From: Marcin Gorzynski Date: Fri, 9 Oct 2026 09:07:51 +0200 Subject: [PATCH] fix(storage/local): write files atomically Create truncated the target and wrote it in place, so a reader that loaded the .fileIndex while the writer saved it could get a partial file (unexpected EOF), and a process exiting mid-save left the index truncated for good. Seen in production: a 380 KB ethlog index cut to its first 40k of 76k entries. The local FS now writes to a hidden temporary file in the same directory and Close syncs it and renames it over the target, so readers see the old or the new file, never part of one. A failed write is discarded and the previous file kept. Other file systems are unchanged; GCS uploads are already atomic. Co-Authored-By: Claude Opus 5.5 (1M context) --- common_test.go | 35 +++++++++++++++ storage/local/local.go | 86 ++++++++++++++++++++++++++++++++++++- storage/local/local_test.go | 84 ++++++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 storage/local/local_test.go diff --git a/common_test.go b/common_test.go index e97063a..bea0fdd 100644 --- a/common_test.go +++ b/common_test.go @@ -312,6 +312,41 @@ func TestFileIndex_Save(t *testing.T) { require.NoError(t, err) } +// A reader loading the index while the writer saves it must never see a +// partial file: the local FS publishes the new index atomically. +func TestFileIndex_ConcurrentSaveAndLoad(t *testing.T) { + ctx := context.Background() + fs := local.NewLocalFS(t.TempDir()) + files := make([]*File, 20000) + for i := range files { + files[i] = &File{FirstBlockNum: uint64(i) + 1, LastBlockNum: uint64(i) + 1} + } + fi := NewFileIndexFromFiles(fs, files) + require.NoError(t, fi.Save(ctx)) + + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < 30; i++ { + if err := fi.Save(ctx); err != nil { + t.Errorf("save: %v", err) + return + } + } + }() + loads := 0 + for { + select { + case <-done: + require.NotZero(t, loads) + return + default: + } + require.NoError(t, NewFileIndex(fs).Load(ctx), "load %d", loads) + loads++ + } +} + func BenchmarkFindInFileIndex(b *testing.B) { benchCase := []struct { NumFiles uint64 diff --git a/storage/local/local.go b/storage/local/local.go index 94ebca6..a476a68 100644 --- a/storage/local/local.go +++ b/storage/local/local.go @@ -1,18 +1,102 @@ package local import ( + "context" + "errors" + "io" "os" + "path/filepath" "github.com/Shopify/go-storage" ) +// createFileMode is the mode of created files, as os.Create gives them +// (0666) under the usual 022 umask. +const createFileMode = os.FileMode(0o644) + type LocalFS struct { storage.FS + + root string } func NewLocalFS(path string) *LocalFS { if len(path) > 0 && path[len(path)-1] != os.PathSeparator { path = path + string(os.PathSeparator) } - return &LocalFS{FS: storage.NewLocalFS(path)} + return &LocalFS{FS: storage.NewLocalFS(path), root: path} +} + +// Create writes the file atomically: the data goes to a hidden temporary file +// next to path, which Close syncs and renames over path. A reader therefore +// sees either the previous file or the complete new one, never a partial +// write, and a crash mid-write leaves the previous file intact. This matters +// for files rewritten in place, such as the .fileIndex. If any write fails, +// Close removes the temporary file and keeps the previous one. +func (l *LocalFS) Create(_ context.Context, path string, options *storage.WriterOptions) (io.WriteCloser, error) { + full := filepath.Join(l.root, path) + dir := filepath.Dir(full) + if err := os.MkdirAll(dir, storage.LocalCreatePathMode); err != nil { + return nil, err + } + tmp, err := os.CreateTemp(dir, "."+filepath.Base(full)+".tmp-*") + if err != nil { + return nil, err + } + if err := tmp.Chmod(createFileMode); err != nil { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + return nil, err + } + return &atomicFile{File: tmp, path: full, options: options}, nil +} + +// atomicFile is a temporary file published at path by Close. +type atomicFile struct { + *os.File + path string + options *storage.WriterOptions + writeErr error + closed bool +} + +func (f *atomicFile) Write(p []byte) (int, error) { + n, err := f.File.Write(p) + if err != nil && f.writeErr == nil { + f.writeErr = err + } + return n, err +} + +func (f *atomicFile) Close() error { + if f.closed { + return nil + } + f.closed = true + tmp := f.File.Name() + if f.writeErr != nil { + _ = f.File.Close() + _ = os.Remove(tmp) + return f.writeErr + } + if err := errors.Join(f.File.Sync(), f.File.Close()); err != nil { + _ = os.Remove(tmp) + return err + } + if err := os.Rename(tmp, f.path); err != nil { + _ = os.Remove(tmp) + return err + } + if f.options != nil { + modTime := f.options.Attributes.ModTime + if !f.options.Attributes.CreationTime.IsZero() { + // There is no way to store the creation time, so it becomes the + // modification time, as in go-storage's local FS. + modTime = f.options.Attributes.CreationTime + } + if !modTime.IsZero() { + return os.Chtimes(f.path, modTime, modTime) + } + } + return nil } diff --git a/storage/local/local_test.go b/storage/local/local_test.go new file mode 100644 index 0000000..a368bb4 --- /dev/null +++ b/storage/local/local_test.go @@ -0,0 +1,84 @@ +package local + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreatePublishesOnClose(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + fs := NewLocalFS(dir) + target := filepath.Join(dir, "a", "b", "file") + + w, err := fs.Create(ctx, "a/b/file", nil) + require.NoError(t, err) + _, err = w.Write([]byte("new")) + require.NoError(t, err) + _, err = os.Stat(target) + require.True(t, os.IsNotExist(err), "nothing is visible at the path before Close") + + require.NoError(t, w.Close()) + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "new", string(got)) + st, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, createFileMode, st.Mode().Perm()) + require.NoError(t, w.Close(), "a second Close is a no-op") + requireNoTemp(t, filepath.Dir(target)) +} + +func TestCreateKeepsPreviousFileUntilClose(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + fs := NewLocalFS(dir) + target := filepath.Join(dir, ".fileIndex") + require.NoError(t, os.WriteFile(target, []byte("old content"), 0o644)) + + w, err := fs.Create(ctx, ".fileIndex", nil) + require.NoError(t, err) + _, err = w.Write([]byte("new")) + require.NoError(t, err) + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "old content", string(got), "a reader sees the previous file, not a truncated one") + + require.NoError(t, w.Close()) + got, err = os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "new", string(got)) + requireNoTemp(t, dir) +} + +func TestCreateDiscardsFailedWrite(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + fs := NewLocalFS(dir) + target := filepath.Join(dir, "file") + require.NoError(t, os.WriteFile(target, []byte("old"), 0o644)) + + w, err := fs.Create(ctx, "file", nil) + require.NoError(t, err) + af := w.(*atomicFile) + require.NoError(t, af.File.Close()) // make the next write fail + _, err = w.Write([]byte("partial")) + require.Error(t, err) + require.Error(t, w.Close(), "Close reports the write error") + + got, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "old", string(got), "a failed write never replaces the file") + requireNoTemp(t, dir) +} + +func requireNoTemp(t *testing.T, dir string) { + t.Helper() + matches, err := filepath.Glob(filepath.Join(dir, ".*.tmp-*")) + require.NoError(t, err) + require.Empty(t, matches, "temporary files are cleaned up") +}