Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,41 @@ func TestFileIndex_Save(t *testing.T) {
require.NoError(t, err)
}

// A reader loading the index while the writer saves it must never see a
// partial file: the local FS publishes the new index atomically.
func TestFileIndex_ConcurrentSaveAndLoad(t *testing.T) {
ctx := context.Background()
fs := local.NewLocalFS(t.TempDir())
files := make([]*File, 20000)
for i := range files {
files[i] = &File{FirstBlockNum: uint64(i) + 1, LastBlockNum: uint64(i) + 1}
}
fi := NewFileIndexFromFiles(fs, files)
require.NoError(t, fi.Save(ctx))

done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 30; i++ {
if err := fi.Save(ctx); err != nil {
t.Errorf("save: %v", err)
return
}
}
}()
loads := 0
for {
select {
case <-done:
require.NotZero(t, loads)
return
default:
}
require.NoError(t, NewFileIndex(fs).Load(ctx), "load %d", loads)
loads++
}
}

func BenchmarkFindInFileIndex(b *testing.B) {
benchCase := []struct {
NumFiles uint64
Expand Down
86 changes: 85 additions & 1 deletion storage/local/local.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,102 @@
package local

import (
"context"
"errors"
"io"
"os"
"path/filepath"

"github.com/Shopify/go-storage"
)

// createFileMode is the mode of created files, as os.Create gives them
// (0666) under the usual 022 umask.
const createFileMode = os.FileMode(0o644)

type LocalFS struct {
storage.FS

root string
}

func NewLocalFS(path string) *LocalFS {
if len(path) > 0 && path[len(path)-1] != os.PathSeparator {
path = path + string(os.PathSeparator)
}
return &LocalFS{FS: storage.NewLocalFS(path)}
return &LocalFS{FS: storage.NewLocalFS(path), root: path}
}

// Create writes the file atomically: the data goes to a hidden temporary file
// next to path, which Close syncs and renames over path. A reader therefore
// sees either the previous file or the complete new one, never a partial
// write, and a crash mid-write leaves the previous file intact. This matters
// for files rewritten in place, such as the .fileIndex. If any write fails,
// Close removes the temporary file and keeps the previous one.
func (l *LocalFS) Create(_ context.Context, path string, options *storage.WriterOptions) (io.WriteCloser, error) {
full := filepath.Join(l.root, path)
dir := filepath.Dir(full)
if err := os.MkdirAll(dir, storage.LocalCreatePathMode); err != nil {
return nil, err
}
tmp, err := os.CreateTemp(dir, "."+filepath.Base(full)+".tmp-*")
if err != nil {
return nil, err
}
if err := tmp.Chmod(createFileMode); err != nil {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
return nil, err
}
return &atomicFile{File: tmp, path: full, options: options}, nil
}

// atomicFile is a temporary file published at path by Close.
type atomicFile struct {
*os.File
path string
options *storage.WriterOptions
writeErr error
closed bool
}

func (f *atomicFile) Write(p []byte) (int, error) {
n, err := f.File.Write(p)
if err != nil && f.writeErr == nil {
f.writeErr = err
}
return n, err
}

func (f *atomicFile) Close() error {
if f.closed {
return nil
}
f.closed = true
tmp := f.File.Name()
if f.writeErr != nil {
_ = f.File.Close()
_ = os.Remove(tmp)
return f.writeErr
}
if err := errors.Join(f.File.Sync(), f.File.Close()); err != nil {
_ = os.Remove(tmp)
return err
}
if err := os.Rename(tmp, f.path); err != nil {
_ = os.Remove(tmp)
return err
}
if f.options != nil {
modTime := f.options.Attributes.ModTime
if !f.options.Attributes.CreationTime.IsZero() {
// There is no way to store the creation time, so it becomes the
// modification time, as in go-storage's local FS.
modTime = f.options.Attributes.CreationTime
}
if !modTime.IsZero() {
return os.Chtimes(f.path, modTime, modTime)
}
}
return nil
}
84 changes: 84 additions & 0 deletions storage/local/local_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package local

import (
"context"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
)

func TestCreatePublishesOnClose(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
fs := NewLocalFS(dir)
target := filepath.Join(dir, "a", "b", "file")

w, err := fs.Create(ctx, "a/b/file", nil)
require.NoError(t, err)
_, err = w.Write([]byte("new"))
require.NoError(t, err)
_, err = os.Stat(target)
require.True(t, os.IsNotExist(err), "nothing is visible at the path before Close")

require.NoError(t, w.Close())
got, err := os.ReadFile(target)
require.NoError(t, err)
require.Equal(t, "new", string(got))
st, err := os.Stat(target)
require.NoError(t, err)
require.Equal(t, createFileMode, st.Mode().Perm())
require.NoError(t, w.Close(), "a second Close is a no-op")
requireNoTemp(t, filepath.Dir(target))
}

func TestCreateKeepsPreviousFileUntilClose(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
fs := NewLocalFS(dir)
target := filepath.Join(dir, ".fileIndex")
require.NoError(t, os.WriteFile(target, []byte("old content"), 0o644))

w, err := fs.Create(ctx, ".fileIndex", nil)
require.NoError(t, err)
_, err = w.Write([]byte("new"))
require.NoError(t, err)
got, err := os.ReadFile(target)
require.NoError(t, err)
require.Equal(t, "old content", string(got), "a reader sees the previous file, not a truncated one")

require.NoError(t, w.Close())
got, err = os.ReadFile(target)
require.NoError(t, err)
require.Equal(t, "new", string(got))
requireNoTemp(t, dir)
}

func TestCreateDiscardsFailedWrite(t *testing.T) {
ctx := context.Background()
dir := t.TempDir()
fs := NewLocalFS(dir)
target := filepath.Join(dir, "file")
require.NoError(t, os.WriteFile(target, []byte("old"), 0o644))

w, err := fs.Create(ctx, "file", nil)
require.NoError(t, err)
af := w.(*atomicFile)
require.NoError(t, af.File.Close()) // make the next write fail
_, err = w.Write([]byte("partial"))
require.Error(t, err)
require.Error(t, w.Close(), "Close reports the write error")

got, err := os.ReadFile(target)
require.NoError(t, err)
require.Equal(t, "old", string(got), "a failed write never replaces the file")
requireNoTemp(t, dir)
}

func requireNoTemp(t *testing.T, dir string) {
t.Helper()
matches, err := filepath.Glob(filepath.Join(dir, ".*.tmp-*"))
require.NoError(t, err)
require.Empty(t, matches, "temporary files are cleaned up")
}
Loading