From d59af3d4591f8ac68184cbf1c0919c55f81912ad Mon Sep 17 00:00:00 2001 From: Clifford Tawiah Date: Thu, 24 Sep 2026 13:53:23 -0400 Subject: [PATCH] feat(sync): persist local prompt workspaces --- go.mod | 3 + go.sum | 5 + internal/sync/fingerprint.go | 5 +- internal/sync/fingerprint_test.go | 39 ++ internal/sync/local/compile.go | 146 ++++++ internal/sync/local/compile_test.go | 243 ++++++++++ internal/sync/local/delete.go | 159 +++++++ internal/sync/local/reference.go | 173 +++++++ internal/sync/local/reference_test.go | 188 ++++++++ internal/sync/local/render.go | 177 ++++++++ internal/sync/local/replace.go | 236 ++++++++++ internal/sync/local/store.go | 251 +++++++++++ internal/sync/local/store_test.go | 424 ++++++++++++++++++ internal/sync/local/variation.go | 291 ++++++++++++ internal/sync/local/variation_test.go | 256 +++++++++++ .../adapters/plain_markdown/plain_markdown.go | 14 +- internal/sync/resource.go | 8 + 17 files changed, 2604 insertions(+), 14 deletions(-) create mode 100644 internal/sync/local/compile.go create mode 100644 internal/sync/local/compile_test.go create mode 100644 internal/sync/local/delete.go create mode 100644 internal/sync/local/reference.go create mode 100644 internal/sync/local/reference_test.go create mode 100644 internal/sync/local/render.go create mode 100644 internal/sync/local/replace.go create mode 100644 internal/sync/local/store.go create mode 100644 internal/sync/local/store_test.go create mode 100644 internal/sync/local/variation.go create mode 100644 internal/sync/local/variation_test.go diff --git a/go.mod b/go.mod index 87c1277d..25483824 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/launchdarkly/ldcli go 1.25 require ( + github.com/adrg/frontmatter v0.2.0 github.com/adrg/xdg v0.5.3 github.com/atotto/clipboard v0.1.4 github.com/blacktop/go-dwarf v1.0.14 @@ -42,6 +43,7 @@ require ( ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -106,4 +108,5 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.42.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 7045f51d..d22e401d 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4= +github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= github.com/adrg/xdg v0.5.3/go.mod h1:nlTsY+NNiCBGCK2tpm09vRqfVzrc2fLmXGpBLF0zlTQ= github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= @@ -648,6 +652,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/sync/fingerprint.go b/internal/sync/fingerprint.go index 1029d40b..263e9347 100644 --- a/internal/sync/fingerprint.go +++ b/internal/sync/fingerprint.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "strings" ) const variationFingerprintSchema = "launchdarkly.config.variation/v1" @@ -26,13 +25,13 @@ func FingerprintVariation(projectKey, lookupKey string, variation Variation) (st } switch normalized.Mode { case VariationModeAgent: - normalized.Instructions = strings.TrimSpace(normalized.Instructions) + normalized.Instructions = NormalizePromptText(normalized.Instructions) normalized.Messages = nil case VariationModeCompletion: normalized.Instructions = "" normalized.Messages = append([]Message(nil), normalized.Messages...) for index := range normalized.Messages { - normalized.Messages[index].Content = strings.TrimSpace(normalized.Messages[index].Content) + normalized.Messages[index].Content = NormalizePromptText(normalized.Messages[index].Content) } } diff --git a/internal/sync/fingerprint_test.go b/internal/sync/fingerprint_test.go index 7ee34329..edc6dcf8 100644 --- a/internal/sync/fingerprint_test.go +++ b/internal/sync/fingerprint_test.go @@ -68,6 +68,45 @@ func TestFingerprintVariationNormalizesRenderedPromptWhitespace(t *testing.T) { require.Equal(t, "\n Be concise. \n", completion.Messages[0].Content) } +func TestFingerprintVariationNormalizesPromptLineEndings(t *testing.T) { + tests := map[string]struct { + windows Variation + unix Variation + }{ + "agent": { + windows: Variation{ + Mode: VariationModeAgent, Key: "agent", Name: "Agent", + Instructions: "First line.\r\nSecond line.\rThird line.", + }, + unix: Variation{ + Mode: VariationModeAgent, Key: "agent", Name: "Agent", + Instructions: "First line.\nSecond line.\nThird line.", + }, + }, + "completion": { + windows: Variation{ + Mode: VariationModeCompletion, Key: "completion", Name: "Completion", + Messages: []Message{{Role: "system", Content: "First line.\r\nSecond line.\rThird line."}}, + }, + unix: Variation{ + Mode: VariationModeCompletion, Key: "completion", Name: "Completion", + Messages: []Message{{Role: "system", Content: "First line.\nSecond line.\nThird line."}}, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + windowsFingerprint, err := FingerprintVariation("project", "config/"+name, test.windows) + require.NoError(t, err) + unixFingerprint, err := FingerprintVariation("project", "config/"+name, test.unix) + require.NoError(t, err) + + require.Equal(t, unixFingerprint, windowsFingerprint) + }) + } +} + func TestValidateDirectAPIVariationSupportsModelConfigVersion(t *testing.T) { base := Variation{Mode: VariationModeAgent, Key: "default", Name: "Default"} diff --git a/internal/sync/local/compile.go b/internal/sync/local/compile.go new file mode 100644 index 00000000..f1226173 --- /dev/null +++ b/internal/sync/local/compile.go @@ -0,0 +1,146 @@ +package local + +import ( + "cmp" + "errors" + "io/fs" + "os" + "path" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// ErrNoDirectory reports that no local sync directory exists. +var ErrNoDirectory = errors.New(".launchdarkly directory not found") + +// ParseError identifies the local file that could not be compiled. +type ParseError struct { + Path string + Err error +} + +// Error includes the repository-relative file that could not be compiled. +func (e ParseError) Error() string { + return e.Path + ": " + e.Err.Error() +} + +// Unwrap exposes the underlying syntax or validation error. +func (e ParseError) Unwrap() error { + return e.Err +} + +// Compile reads local resources from an arbitrary filesystem. +func Compile(fsys fs.FS) ([]syncdomain.SyncedResource, error) { + return compile(fsys, func(reference Reference) ([]byte, error) { + return readReferenceFromFS(fsys, reference) + }) +} + +// CompileWorkspace compiles local resources and safely resolves references +// within the Git repository. +func CompileWorkspace(repositoryRoot string) ([]syncdomain.SyncedResource, error) { + return compile(os.DirFS(repositoryRoot), func(reference Reference) ([]byte, error) { + return readWorkspaceReference(repositoryRoot, reference) + }) +} + +// compile walks every managed project and delegates reference loading to the +// caller so tests and real workspaces share the same parser. +func compile(fsys fs.FS, readReference func(Reference) ([]byte, error)) ([]syncdomain.SyncedResource, error) { + entries, err := fs.ReadDir(fsys, syncdomain.RootDir) + if errors.Is(err, fs.ErrNotExist) { + return nil, ErrNoDirectory + } + if err != nil { + return nil, err + } + + var resources []syncdomain.SyncedResource + + // Directories immediately below .launchdarkly are project scopes. Files at + // the root, including the manifest, are handled by their owning packages. + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + variations, err := compileProjectVariations(fsys, entry.Name(), readReference) + if err != nil { + return nil, err + } + + resources = append(resources, variations...) + } + + slices.SortFunc(resources, compareResources) + + return resources, nil +} + +// compileProjectVariations turns every supported wrapper in one project into +// the common resource representation consumed by reconciliation. +func compileProjectVariations( + fsys fs.FS, + projectKey string, + readReference func(Reference) ([]byte, error), +) ([]syncdomain.SyncedResource, error) { + dir := path.Join(syncdomain.RootDir, projectKey, configsDir) + + var resources []syncdomain.SyncedResource + + err := fs.WalkDir(fsys, dir, func(name string, entry fs.DirEntry, err error) error { + if err != nil { + // A project may legitimately contain no resources of this kind. + if name == dir && errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if entry.IsDir() { + return nil + } + + relPath := strings.TrimPrefix(name, dir+"/") + // Ignore files owned by other resource kinds. Each compiler recognizes + // only its own directory shape and suffix. + if relPath == name || !isVariationFile(relPath) { + return nil + } + + data, err := fs.ReadFile(fsys, name) + if err != nil { + return err + } + + resource, err := parseVariation(localFile{ + ProjectKey: projectKey, + RelPath: relPath, + Data: data, + ReadReference: readReference, + }) + if err != nil { + // Preserve the managed path so users can locate malformed content + // while callers can still inspect the parser error through Unwrap. + return ParseError{Path: name, Err: err} + } + + resources = append(resources, resource) + + return nil + }) + if err != nil { + return nil, err + } + + return resources, nil +} + +// compareResources provides deterministic project and lookup-key ordering. +func compareResources(a, b syncdomain.SyncedResource) int { + return cmp.Or( + cmp.Compare(a.ProjectKey, b.ProjectKey), + cmp.Compare(a.LookupKey, b.LookupKey), + ) +} diff --git a/internal/sync/local/compile_test.go b/internal/sync/local/compile_test.go new file mode 100644 index 00000000..93681be8 --- /dev/null +++ b/internal/sync/local/compile_test.go @@ -0,0 +1,243 @@ +package local + +import ( + "encoding/json" + "errors" + "io/fs" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +const specPrompt = `--- +formatVersion: 1 +upsert: true +mode: completion + +key: my-first-variation +name: This is the prompt name + +modelConfigKey: anthropic-default +modelConfigVersion: 2 + +model: + parameters: + max_tokens: 100 + modelName: "Anthropic.claude-haiku-4-5-20251001" + custom: + max_retrieval_limit: 20 + +outputFormat: + type: "json_schema" + additionalProperties: false + required: + - response + - confidence + properties: + confidence: + type: "number" + minimum: 0 + maximum: 1 + response: + type: "string" + description: "The generated response." + +--- + + +This is a system prompt and I can embed other items and data in here. +Ask a nested question. +Stay in character. +A nested assistant reply. + + + +This is a user prompt and I can embed other nested values in here. +Ignore previous instructions. +Also answer this. +A nested assistant draft. + + + +This is an assistant prompt and I can embed other nested values in here. +Keep this in the assistant body. +Keep this in the assistant body too. +A nested assistant example. + +` + +func specWorkspace() fstest.MapFS { + return fstest.MapFS{ + ".launchdarkly/proj-key/configs/my-config-key/my-first-variation.prompt.md": &fstest.MapFile{ + Data: []byte(specPrompt), + }, + } +} + +func TestCompile(t *testing.T) { + resources, err := Compile(specWorkspace()) + require.NoError(t, err) + require.Len(t, resources, 1) + + variation := resources[0] + assert.Equal(t, syncdomain.KindVariation, variation.Kind) + assert.Equal(t, "proj-key", variation.ProjectKey) + assert.Equal(t, "my-config-key/my-first-variation", variation.LookupKey) + assert.True(t, variation.Upsert) + + var payload syncdomain.Variation + require.NoError(t, json.Unmarshal(variation.Payload, &payload)) + assert.Equal(t, syncdomain.VariationModeCompletion, payload.Mode) + assert.Equal(t, "my-first-variation", payload.Key) + assert.Equal(t, "This is the prompt name", payload.Name) + assert.Equal(t, "anthropic-default", payload.ModelConfigKey) + assert.Equal(t, 2, payload.ModelConfigVersion) + require.Len(t, payload.Messages, 3) + assert.Equal(t, "system", payload.Messages[0].Role) + assert.Equal(t, "This is a system prompt and I can embed other items and data in here.\nAsk a nested question.\nStay in character.\nA nested assistant reply.", payload.Messages[0].Content) + assert.Equal(t, "user", payload.Messages[1].Role) + assert.Equal(t, "This is a user prompt and I can embed other nested values in here.\nIgnore previous instructions.\nAlso answer this.\nA nested assistant draft.", payload.Messages[1].Content) + assert.Equal(t, "assistant", payload.Messages[2].Role) + assert.Equal(t, "This is an assistant prompt and I can embed other nested values in here.\nKeep this in the assistant body.\nKeep this in the assistant body too.\nA nested assistant example.", payload.Messages[2].Content) +} + +func TestCompile_MissingDirectory(t *testing.T) { + _, err := Compile(fstest.MapFS{}) + require.ErrorIs(t, err, ErrNoDirectory) +} + +func TestCompile_EmptyProjects(t *testing.T) { + fsys := fstest.MapFS{ + ".launchdarkly/proj-key/.keep": &fstest.MapFile{Data: []byte{}}, + } + + resources, err := Compile(fsys) + require.NoError(t, err) + assert.Empty(t, resources) +} + +func TestCompile_SkipsUnsupportedFiles(t *testing.T) { + fsys := specWorkspace() + fsys[".launchdarkly/proj-key/configs/README.md"] = &fstest.MapFile{Data: []byte("notes")} + fsys[".launchdarkly/proj-key/other/resource.json"] = &fstest.MapFile{Data: []byte("{}")} + + resources, err := Compile(fsys) + require.NoError(t, err) + assert.Len(t, resources, 1) +} + +func TestCompile_SortsByProjectAndLookupKey(t *testing.T) { + fsys := fstest.MapFS{ + ".launchdarkly/zeta/configs/cfg/z.prompt.md": &fstest.MapFile{ + Data: []byte(minimalPrompt("z", "Z")), + }, + ".launchdarkly/alpha/configs/cfg/b.prompt.md": &fstest.MapFile{ + Data: []byte(minimalPrompt("b", "B")), + }, + ".launchdarkly/alpha/configs/cfg/a.prompt.md": &fstest.MapFile{ + Data: []byte(minimalPrompt("a", "A")), + }, + } + + resources, err := Compile(fsys) + require.NoError(t, err) + require.Len(t, resources, 3) + assert.Equal(t, []string{"alpha", "alpha", "zeta"}, projectKeys(resources)) + assert.Equal(t, []string{"cfg/a", "cfg/b", "cfg/z"}, lookupKeys(resources)) +} + +func TestCompile_ReturnsFileMissingDuringWalk(t *testing.T) { + const missingPath = ".launchdarkly/proj-key/configs/cfg/z.prompt.md" + fsys := missingReadFileFS{ + FS: fstest.MapFS{ + ".launchdarkly/proj-key/configs/cfg/a.prompt.md": &fstest.MapFile{ + Data: []byte(minimalPrompt("a", "A")), + }, + missingPath: &fstest.MapFile{ + Data: []byte(minimalPrompt("z", "Z")), + }, + }, + path: missingPath, + } + + _, err := Compile(fsys) + + require.ErrorIs(t, err, fs.ErrNotExist) + var pathErr *fs.PathError + require.ErrorAs(t, err, &pathErr) + assert.Equal(t, missingPath, pathErr.Path) +} + +func TestCompile_ParseErrorIncludesPath(t *testing.T) { + fsys := fstest.MapFS{ + ".launchdarkly/proj-key/configs/my-config-key/wrong-name.prompt.md": &fstest.MapFile{ + Data: []byte(minimalPrompt("my-first-variation", "Name")), + }, + } + + _, err := Compile(fsys) + require.Error(t, err) + + var parseErr ParseError + require.ErrorAs(t, err, &parseErr) + assert.Equal(t, ".launchdarkly/proj-key/configs/my-config-key/wrong-name.prompt.md", parseErr.Path) + assert.ErrorContains(t, parseErr.Err, `key "my-first-variation" does not match filename "wrong-name"`) +} + +func TestCompile_MissingFrontMatter(t *testing.T) { + fsys := fstest.MapFS{ + ".launchdarkly/proj-key/configs/cfg/var.prompt.md": &fstest.MapFile{ + Data: []byte("just a prompt"), + }, + } + + _, err := Compile(fsys) + require.ErrorContains(t, err, "missing YAML front matter") +} + +func TestErrNoDirectory_Is(t *testing.T) { + _, err := Compile(fstest.MapFS{ + "README.md": &fstest.MapFile{Data: []byte("nope")}, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrNoDirectory)) + assert.False(t, errors.Is(err, fs.ErrNotExist)) +} + +func minimalPrompt(key, name string) string { + return "---\nformatVersion: 1\nmode: completion\nkey: " + key + "\nname: " + name + "\n---\n" +} + +type missingReadFileFS struct { + fs.FS + path string +} + +func (f missingReadFileFS) ReadFile(name string) ([]byte, error) { + if name == f.path { + return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} + } + return fs.ReadFile(f.FS, name) +} + +func projectKeys(resources []syncdomain.SyncedResource) []string { + keys := make([]string, len(resources)) + for index, resource := range resources { + keys[index] = resource.ProjectKey + } + + return keys +} + +func lookupKeys(resources []syncdomain.SyncedResource) []string { + keys := make([]string, len(resources)) + for index, resource := range resources { + keys[index] = resource.LookupKey + } + + return keys +} diff --git a/internal/sync/local/delete.go b/internal/sync/local/delete.go new file mode 100644 index 00000000..9551577d --- /dev/null +++ b/internal/sync/local/delete.go @@ -0,0 +1,159 @@ +package local + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +type stagedDeletion struct { + relativePath string + originalPath string + backupPath string +} + +// DeleteVariations preflights and stages a batch before removing any wrapper +// permanently, allowing staging failures to restore the original files. +func (store Store) DeleteVariations(resources []VariationDeletion) ([]string, error) { + deletions, err := store.prepareDeletions(resources) + if err != nil { + return nil, err + } + + // Move the complete batch to same-directory backups before removing + // anything permanently. A staging failure can therefore restore every file. + if err := stageDeletions(deletions); err != nil { + return nil, err + } + if err := commitDeletions(store.root, deletions); err != nil { + return nil, err + } + return deletionPaths(deletions), nil +} + +// prepareDeletions validates the complete batch and rejects duplicate paths +// before filesystem state changes. +func (store Store) prepareDeletions(resources []VariationDeletion) ([]stagedDeletion, error) { + deletions := make([]stagedDeletion, 0, len(resources)) + seenPaths := make(map[string]struct{}, len(resources)) + + for _, resource := range resources { + absolutePath, err := store.variationPath(resource.ProjectKey, resource.ConfigKey, resource.VariationKey) + if err != nil { + return nil, err + } + // Use Lstat so the regular-file check rejects symlink wrappers rather + // than following them to a file outside the managed workspace. + info, err := os.Lstat(absolutePath) + if err != nil { + return nil, fmt.Errorf("inspect variation %s: %w", resource.VariationKey, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("variation %s is not a regular file", resource.VariationKey) + } + if _, duplicate := seenPaths[absolutePath]; duplicate { + return nil, fmt.Errorf("variation %q was selected more than once", resource.VariationKey) + } + seenPaths[absolutePath] = struct{}{} + relativePath, err := filepath.Rel(store.root, absolutePath) + if err != nil { + return nil, fmt.Errorf("resolve variation %s: %w", resource.VariationKey, err) + } + deletions = append(deletions, stagedDeletion{ + relativePath: filepath.ToSlash(relativePath), + originalPath: absolutePath, + }) + } + return deletions, nil +} + +// stageDeletions renames wrappers to same-directory backups and restores prior +// renames if any later rename fails. +func stageDeletions(deletions []stagedDeletion) error { + var staged []stagedDeletion + for index := range deletions { + backupPath, err := reserveBackupPath(deletions[index].originalPath) + if err == nil { + err = os.Rename(deletions[index].originalPath, backupPath) + } + if err != nil { + return errors.Join(fmt.Errorf("stage deletion %s: %w", deletions[index].relativePath, err), rollbackDeletions(staged)) + } + + deletions[index].backupPath = backupPath + staged = append(staged, deletions[index]) + } + return nil +} + +// reserveBackupPath obtains a collision-free backup name beside a wrapper. +func reserveBackupPath(originalPath string) (string, error) { + temp, err := os.CreateTemp(filepath.Dir(originalPath), "."+filepath.Base(originalPath)+".deleted-") + if err != nil { + return "", err + } + + backupPath := temp.Name() + // CreateTemp reserves a collision-free name. Removing the placeholder lets + // Rename move the original into that exact same-directory location. + if err := temp.Close(); err != nil { + _ = os.Remove(backupPath) + return "", err + } + if err := os.Remove(backupPath); err != nil { + return "", err + } + return backupPath, nil +} + +// commitDeletions removes staged backups and restores every remaining backup +// if cleanup cannot continue. +func commitDeletions(root string, deletions []stagedDeletion) error { + for index, deletion := range deletions { + if err := os.Remove(deletion.backupPath); err != nil { + // Backups deleted earlier are already committed. Restore every + // remaining backup so no additional resources are lost. + return errors.Join( + fmt.Errorf("finish deleting variation %s: %w", deletion.relativePath, err), + rollbackDeletions(deletions[index:]), + ) + } + removeEmptyParentsThroughRoot(root, filepath.Dir(deletion.originalPath)) + } + return nil +} + +// deletionPaths returns the stable repository-relative paths reported to callers. +func deletionPaths(deletions []stagedDeletion) []string { + paths := make([]string, len(deletions)) + for index, deletion := range deletions { + paths[index] = deletion.relativePath + } + return paths +} + +// rollbackDeletions restores staged wrappers in reverse order. +func rollbackDeletions(deletions []stagedDeletion) error { + var rollbackErr error + for index := len(deletions) - 1; index >= 0; index-- { + if err := os.Rename(deletions[index].backupPath, deletions[index].originalPath); err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf("restore variation %s: %w", deletions[index].relativePath, err)) + } + } + return rollbackErr +} + +// removeEmptyParentsThroughRoot removes empty variation, config, and project +// directories. It also removes the .launchdarkly root when the store is empty. +func removeEmptyParentsThroughRoot(root, current string) { + for { + if err := os.Remove(current); err != nil { + return + } + if current == root { + return + } + current = filepath.Dir(current) + } +} diff --git a/internal/sync/local/reference.go b/internal/sync/local/reference.go new file mode 100644 index 00000000..b9dd287d --- /dev/null +++ b/internal/sync/local/reference.go @@ -0,0 +1,173 @@ +package local + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// Reference identifies an external source file and the adapter format that +// converts it into a synchronized resource. +type Reference struct { + File string `yaml:"file"` + Format string `yaml:"format"` +} + +// NewReference validates a user-supplied path and converts it to the +// repository-relative path stored in frontmatter. +func NewReference(repositoryRoot, workingDirectory, file, format string) (Reference, error) { + if file == "" { + return Reference{}, fmt.Errorf("linked file path is required") + } + if !filepath.IsAbs(file) { + file = filepath.Join(workingDirectory, file) + } + // Compare resolved paths, not their textual spelling. Otherwise a path that + // appears to be inside the repository could escape through a symlink. + target, err := filepath.EvalSymlinks(file) + if err != nil { + return Reference{}, fmt.Errorf("resolve linked file %q: %w", file, err) + } + root, err := filepath.EvalSymlinks(repositoryRoot) + if err != nil { + return Reference{}, fmt.Errorf("resolve repository root: %w", err) + } + relative, err := filepath.Rel(root, target) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return Reference{}, fmt.Errorf("linked file must be inside the Git repository") + } + + reference := Reference{File: filepath.ToSlash(relative), Format: format} + if _, err := resolveReferencePath(root, reference); err != nil { + return Reference{}, err + } + return reference, nil +} + +// readReferenceFromFS reads a validated reference from an abstract filesystem. +func readReferenceFromFS(fsys fs.FS, reference Reference) ([]byte, error) { + if err := validateReference(reference); err != nil { + return nil, err + } + data, err := fs.ReadFile(fsys, reference.File) + if err != nil { + return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) + } + return data, nil +} + +// readWorkspaceReference resolves symlinks before reading a source from disk. +func readWorkspaceReference(repositoryRoot string, reference Reference) ([]byte, error) { + target, err := resolveReferencePath(repositoryRoot, reference) + if err != nil { + return nil, err + } + data, err := os.ReadFile(target) + if err != nil { + return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) + } + return data, nil +} + +// resolveReferencePath proves that both the declared path and its resolved +// symlink target remain inside the repository. +func resolveReferencePath(repositoryRoot string, reference Reference) (string, error) { + if err := validateReference(reference); err != nil { + return "", err + } + + // Re-resolve both sides on every read. A symlink may have changed since the + // wrapper was created, so validation at link time is not sufficient. + root, err := filepath.EvalSymlinks(repositoryRoot) + if err != nil { + return "", fmt.Errorf("resolve repository root: %w", err) + } + target, err := filepath.EvalSymlinks(filepath.Join(root, filepath.FromSlash(reference.File))) + if err != nil { + return "", fmt.Errorf("resolve referenced file %q: %w", reference.File, err) + } + relative, err := filepath.Rel(root, target) + if err != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("referenced file %q resolves outside the Git repository", reference.File) + } + info, err := os.Stat(target) + if err != nil { + return "", fmt.Errorf("inspect referenced file %q: %w", reference.File, err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("referenced file %q is not a regular file", reference.File) + } + return target, nil +} + +// validateReference checks the portable, repository-relative reference syntax. +func validateReference(reference Reference) error { + switch { + case reference.File == "": + return fmt.Errorf("ref.file is required") + case reference.Format == "": + return fmt.Errorf("ref.format is required") + case !fs.ValidPath(reference.File): + return fmt.Errorf("ref.file %q must be a repository-relative path", reference.File) + case reference.File == syncdomain.RootDir || strings.HasPrefix(reference.File, syncdomain.RootDir+"/"): + return fmt.Errorf("ref.file %q must be outside %s", reference.File, syncdomain.RootDir) + case path.Clean(reference.File) != reference.File: + return fmt.Errorf("ref.file %q must be a clean repository-relative path", reference.File) + default: + return nil + } +} + +// SourceFiles returns managed files and external references that can affect the +// current sync plan. Invalid managed files remain watched so fixing them triggers sync. +func SourceFiles(repositoryRoot string) ([]string, error) { + managedRoot := filepath.Join(repositoryRoot, syncdomain.RootDir) + var files []string + err := filepath.WalkDir(managedRoot, func(filePath string, entry os.DirEntry, walkErr error) error { + if errors.Is(walkErr, os.ErrNotExist) { + return nil + } + if walkErr != nil { + return walkErr + } + if entry.IsDir() || !strings.HasSuffix(entry.Name(), variationFileSuffix) { + return nil + } + + // Add the managed file before attempting to parse it. A malformed file + // must remain watched so correcting its syntax can trigger another sync. + relative, err := filepath.Rel(repositoryRoot, filePath) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(relative)) + + content, err := os.ReadFile(filePath) + if err != nil { + return nil + } + // Reference discovery is best effort. The compiler will report detailed + // syntax errors; the watcher only needs valid references it can follow. + var metadata variationFrontMatter + if _, err := parseYAMLFrontMatter(content, &metadata); err == nil && metadata.Ref != nil && validateReference(*metadata.Ref) == nil { + files = append(files, metadata.Ref.File) + } + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("find sync source files: %w", err) + } + + slices.Sort(files) + return slices.Compact(files), nil +} diff --git a/internal/sync/local/reference_test.go b/internal/sync/local/reference_test.go new file mode 100644 index 00000000..9c0054e1 --- /dev/null +++ b/internal/sync/local/reference_test.go @@ -0,0 +1,188 @@ +package local + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +func TestNewReferenceStoresRepositoryRelativePath(t *testing.T) { + root := t.TempDir() + workingDirectory := filepath.Join(root, "app") + require.NoError(t, os.MkdirAll(filepath.Join(workingDirectory, "prompts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(workingDirectory, "prompts", "support.md"), []byte("Help"), 0o644)) + + reference, err := NewReference(root, workingDirectory, "prompts/support.md", syncreference.PlainMarkdown) + + require.NoError(t, err) + require.Equal(t, Reference{File: "app/prompts/support.md", Format: syncreference.PlainMarkdown}, reference) +} + +func TestNewReferenceRejectsUnsafePaths(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "prompt.md") + require.NoError(t, os.WriteFile(outside, []byte("Help"), 0o644)) + + _, err := NewReference(root, root, outside, syncreference.PlainMarkdown) + require.ErrorContains(t, err, "inside the Git repository") + + managed := filepath.Join(root, syncdomain.RootDir, "prompt.md") + require.NoError(t, os.MkdirAll(filepath.Dir(managed), 0o755)) + require.NoError(t, os.WriteFile(managed, []byte("Help"), 0o644)) + _, err = NewReference(root, root, managed, syncreference.PlainMarkdown) + require.ErrorContains(t, err, "must be outside") +} + +func TestNewReferenceRejectsSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "prompt.md") + require.NoError(t, os.WriteFile(outside, []byte("Help"), 0o644)) + link := filepath.Join(root, "prompt.md") + require.NoError(t, os.Symlink(outside, link)) + + _, err := NewReference(root, root, link, syncreference.PlainMarkdown) + + require.ErrorContains(t, err, "inside the Git repository") +} + +func TestCompileWorkspaceReadsLinkedPrompt(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "prompts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "prompts", "support.md"), []byte("Be helpful.\n"), 0o644)) + _, err := NewStore(root).Add([]VariationFile{{ + ProjectKey: "project", + ConfigKey: "support", + Upsert: true, + Ref: &Reference{File: "prompts/support.md", Format: syncreference.PlainMarkdown}, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, + Key: "default", + Name: "Default", + }, + }}) + require.NoError(t, err) + + resources, err := CompileWorkspace(root) + + require.NoError(t, err) + require.Len(t, resources, 1) + var variation syncdomain.Variation + require.NoError(t, json.Unmarshal(resources[0].Payload, &variation)) + require.Equal(t, "Be helpful.", variation.Instructions) +} + +func TestReplaceVariationsUpdatesLinkedWrapperAndPrompt(t *testing.T) { + root := t.TempDir() + referencePath := filepath.Join(root, "prompts", "support.md") + require.NoError(t, os.MkdirAll(filepath.Dir(referencePath), 0o755)) + require.NoError(t, os.WriteFile(referencePath, []byte("Old prompt\n"), 0o640)) + store := NewStore(root) + reference := &Reference{File: "prompts/support.md", Format: syncreference.PlainMarkdown} + _, err := store.Add([]VariationFile{{ + ProjectKey: "project", ConfigKey: "support", Upsert: true, Ref: reference, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Old", Instructions: "Old prompt", + }, + }}) + require.NoError(t, err) + + serverVariation := syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, Key: "default", Name: "New", + Instructions: "First line.\r\nSecond line.\rThird line.", + } + paths, err := store.ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", + ConfigKey: "support", + Variation: serverVariation, + }}) + + require.NoError(t, err) + require.Equal(t, []string{"project/configs/support/default.prompt.md"}, paths) + content, err := os.ReadFile(referencePath) + require.NoError(t, err) + require.Equal(t, "First line.\nSecond line.\nThird line.\n", string(content)) + info, err := os.Stat(referencePath) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o640), info.Mode().Perm()) + wrapper, err := os.ReadFile(filepath.Join(root, syncdomain.RootDir, paths[0])) + require.NoError(t, err) + require.Contains(t, string(wrapper), "name: New") + require.Contains(t, string(wrapper), "file: prompts/support.md") + + compiled, err := CompileWorkspace(root) + require.NoError(t, err) + resource := requireVariationResource(t, compiled, "support/default") + var localVariation syncdomain.Variation + require.NoError(t, json.Unmarshal(resource.Payload, &localVariation)) + serverFingerprint, err := syncdomain.FingerprintVariation("project", "support/default", serverVariation) + require.NoError(t, err) + localFingerprint, err := syncdomain.FingerprintVariation("project", "support/default", localVariation) + require.NoError(t, err) + require.Equal(t, serverFingerprint, localFingerprint) +} + +func TestReplaceVariationsDoesNotChangeLinkedFilesWhenServerPromptIsNotRepresentable(t *testing.T) { + root := t.TempDir() + referencePath := filepath.Join(root, "prompt.md") + require.NoError(t, os.WriteFile(referencePath, []byte("Old prompt\n"), 0o644)) + store := NewStore(root) + _, err := store.Add([]VariationFile{{ + ProjectKey: "project", ConfigKey: "support", + Ref: &Reference{File: "prompt.md", Format: syncreference.PlainMarkdown}, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeCompletion, Key: "default", Name: "Old", + Messages: []syncdomain.Message{{Role: "system", Content: "Old prompt"}}, + }, + }}) + require.NoError(t, err) + wrapperPath := filepath.Join(root, syncdomain.RootDir, "project", configsDir, "support", "default"+variationFileSuffix) + originalWrapper, err := os.ReadFile(wrapperPath) + require.NoError(t, err) + + _, err = store.ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", ConfigKey: "support", + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeCompletion, Key: "default", Name: "New", + Messages: []syncdomain.Message{ + {Role: "system", Content: "System"}, + {Role: "user", Content: "User"}, + }, + }, + }}) + + require.ErrorContains(t, err, "at most one system message") + content, readErr := os.ReadFile(referencePath) + require.NoError(t, readErr) + require.Equal(t, "Old prompt\n", string(content)) + wrapper, readErr := os.ReadFile(wrapperPath) + require.NoError(t, readErr) + require.Equal(t, originalWrapper, wrapper) +} + +func TestSourceFilesIncludesWrappersAndReferences(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, "prompts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "prompts", "support.md"), []byte("Help"), 0o644)) + _, err := NewStore(root).Add([]VariationFile{{ + ProjectKey: "project", ConfigKey: "support", + Ref: &Reference{File: "prompts/support.md", Format: syncreference.PlainMarkdown}, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, Key: "default", Name: "Default", + }, + }}) + require.NoError(t, err) + + files, err := SourceFiles(root) + + require.NoError(t, err) + require.Equal(t, []string{ + ".launchdarkly/project/configs/support/default.prompt.md", + "prompts/support.md", + }, files) +} diff --git a/internal/sync/local/render.go b/internal/sync/local/render.go new file mode 100644 index 00000000..df6f5e85 --- /dev/null +++ b/internal/sync/local/render.go @@ -0,0 +1,177 @@ +package local + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + "gopkg.in/yaml.v3" +) + +// RenderVariations validates and renders variation files without writing them. +func (store Store) RenderVariations(resources []VariationFile) ([]RenderedVariationFile, error) { + rendered := make([]RenderedVariationFile, 0, len(resources)) + seenPaths := make(map[string]struct{}, len(resources)) + + for _, resource := range resources { + absolutePath, err := store.variationPath(resource.ProjectKey, resource.ConfigKey, resource.Variation.Key) + if err != nil { + return nil, err + } + if _, duplicate := seenPaths[absolutePath]; duplicate { + return nil, fmt.Errorf("variation %q was selected more than once", resource.Variation.Key) + } + seenPaths[absolutePath] = struct{}{} + + content, err := marshalVariationFile(resource) + if err != nil { + return nil, err + } + rendered = append(rendered, RenderedVariationFile{ + Path: filepath.ToSlash(strings.TrimPrefix(absolutePath, store.root+string(filepath.Separator))), + Content: content, + }) + } + return rendered, nil +} + +// createVariations writes a prevalidated batch and rolls back files created +// before the first failure. +func (store Store) createVariations(resources []VariationFile) ([]string, error) { + // Render the complete batch first so validation failures cannot leave a + // partially created workspace. + rendered, err := store.RenderVariations(resources) + if err != nil { + return nil, err + } + + var createdPaths []string + for _, file := range rendered { + absolutePath := filepath.Join(store.root, filepath.FromSlash(file.Path)) + if err := createFile(absolutePath, file.Content); err != nil { + // Creates are independent filesystem operations. Remove earlier + // files in reverse order to recover the pre-call state. + for index := len(createdPaths) - 1; index >= 0; index-- { + _ = os.Remove(filepath.Join(store.root, filepath.FromSlash(createdPaths[index]))) + } + return nil, errors.Join(err, store.RemoveEmptyDirectories()) + } + createdPaths = append(createdPaths, file.Path) + } + return createdPaths, nil +} + +// marshalVariationFile converts the canonical variation into front matter and +// the mode-specific prompt body used by local wrapper files. +func marshalVariationFile(resource VariationFile) ([]byte, error) { + if !resource.Variation.Mode.Valid() { + return nil, fmt.Errorf("variation %q has unsupported mode %q", resource.Variation.Key, resource.Variation.Mode) + } + switch resource.Variation.Mode { + case syncdomain.VariationModeAgent: + if len(resource.Variation.Messages) != 0 { + return nil, fmt.Errorf("agent variation %q cannot contain messages", resource.Variation.Key) + } + case syncdomain.VariationModeCompletion: + if resource.Variation.Instructions != "" { + return nil, fmt.Errorf("completion variation %q cannot contain instructions", resource.Variation.Key) + } + } + + var frontMatter bytes.Buffer + encoder := yaml.NewEncoder(&frontMatter) + encoder.SetIndent(2) + err := encoder.Encode(variationFrontMatter{ + FormatVersion: 1, + Upsert: resource.Upsert, + Ref: resource.Ref, + Variation: resource.Variation, + }) + if err != nil { + return nil, fmt.Errorf("marshal variation %q: %w", resource.Variation.Key, err) + } + if err := encoder.Close(); err != nil { + return nil, fmt.Errorf("marshal variation %q: %w", resource.Variation.Key, err) + } + + var file bytes.Buffer + file.WriteString("---\n") + file.Write(frontMatter.Bytes()) + file.WriteString("---\n") + if resource.Ref != nil { + // Referenced wrappers contain metadata only. Keeping the body empty + // prevents two local sources from competing for the same content. + if err := validateReference(*resource.Ref); err != nil { + return nil, err + } + return file.Bytes(), nil + } + if resource.Variation.Mode == syncdomain.VariationModeAgent { + if instructions := syncdomain.NormalizePromptText(resource.Variation.Instructions); instructions != "" { + _, _ = fmt.Fprintf(&file, "\n%s\n", instructions) + } + return file.Bytes(), nil + } + + for _, message := range resource.Variation.Messages { + if !validMessageRole(message.Role) { + return nil, fmt.Errorf("variation %q has unsupported message role %q", resource.Variation.Key, message.Role) + } + content := escapeMessageContent(syncdomain.NormalizePromptText(message.Content), message.Role) + _, _ = fmt.Fprintf(&file, "\n<%s>\n%s\n\n", message.Role, content, message.Role) + } + return file.Bytes(), nil +} + +// validMessageRole reports whether the wrapper syntax supports a role. +func validMessageRole(role string) bool { + return role == "system" || role == "user" || role == "assistant" +} + +// escapeMessageContent protects literal role delimiters and backslashes so a +// rendered completion prompt can always be parsed back losslessly. +func escapeMessageContent(content, role string) string { + content = strings.ReplaceAll(content, `\`, `\\`) + content = strings.ReplaceAll(content, "<"+role+">", `<\`+role+">") + return strings.ReplaceAll(content, "", `<\/`+role+">") +} + +// createFile stages content beside its destination and hard-links it into +// place, which guarantees an existing wrapper is never overwritten. +func createFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("create variation directory: %w", err) + } + + temp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-") + if err != nil { + return fmt.Errorf("stage variation %s: %w", filepath.Base(path), err) + } + tempPath := temp.Name() + defer func() { _ = os.Remove(tempPath) }() + + // The temporary file is fully written before publication. A hard link is + // then an atomic create-if-absent operation on the same filesystem. + if err := temp.Chmod(0o644); err != nil { + _ = temp.Close() + return fmt.Errorf("set variation permissions: %w", err) + } + if _, err := temp.Write(data); err != nil { + _ = temp.Close() + return fmt.Errorf("write variation %s: %w", filepath.Base(path), err) + } + if err := temp.Close(); err != nil { + return fmt.Errorf("close variation %s: %w", filepath.Base(path), err) + } + if err := os.Link(tempPath, path); err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("%w: %s", ErrVariationExists, path) + } + return fmt.Errorf("create variation %s: %w", filepath.Base(path), err) + } + return nil +} diff --git a/internal/sync/local/replace.go b/internal/sync/local/replace.go new file mode 100644 index 00000000..5f4a7f8a --- /dev/null +++ b/internal/sync/local/replace.go @@ -0,0 +1,236 @@ +package local + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" +) + +type stagedVariation struct { + relativePath string + destinationPath string + originalContent []byte + replacementContent []byte + mode os.FileMode + stagedPath string + report bool +} + +// ReplaceVariations transactionally replaces a batch of wrappers and +// referenced files, rolling back committed paths if a later rename fails. +func (store Store) ReplaceVariations(resources []VariationReplacement) ([]string, error) { + replacements, err := store.prepareReplacements(resources) + if err != nil { + return nil, err + } + if err := stageReplacements(replacements); err != nil { + return nil, err + } + defer removeStagedVariations(replacements) + + // Staging can take long enough for an editor to change a source file. + // Recheck every source before replacing any of them. + if err := verifyReplacementSources(replacements); err != nil { + return nil, err + } + if err := commitReplacements(replacements); err != nil { + return nil, err + } + return replacementPaths(replacements), nil +} + +// prepareReplacements renders every destination before any file is changed. +func (store Store) prepareReplacements(resources []VariationReplacement) ([]stagedVariation, error) { + replacements := make([]stagedVariation, 0, len(resources)) + seenPaths := make(map[string]struct{}, len(resources)) + + for _, resource := range resources { + existing, err := store.inspectVariation(resource.ProjectKey, resource.ConfigKey, resource.Variation.Key) + if err != nil { + return nil, err + } + if _, duplicate := seenPaths[existing.absolutePath]; duplicate { + return nil, fmt.Errorf("variation %q was selected more than once", resource.Variation.Key) + } + seenPaths[existing.absolutePath] = struct{}{} + + // Upsert and ref describe the local mapping, not server state. Preserve + // them while replacing only the synchronized variation fields. + content, err := marshalVariationFile(VariationFile{ + ProjectKey: resource.ProjectKey, + ConfigKey: resource.ConfigKey, + Upsert: existing.frontMatter.Upsert, + Ref: existing.frontMatter.Ref, + Variation: resource.Variation, + }) + if err != nil { + return nil, err + } + replacements = append(replacements, stagedVariation{ + relativePath: existing.relativePath, + destinationPath: existing.absolutePath, + originalContent: existing.content, + replacementContent: content, + mode: existing.mode, + report: true, + }) + + if existing.frontMatter.Ref == nil { + continue + } + // Add a linked wrapper and its external source to the same staged + // transaction so a commit failure rolls back both paths. + reference := *existing.frontMatter.Ref + referencePath, err := resolveReferencePath(store.repositoryRoot, reference) + if err != nil { + return nil, err + } + if _, duplicate := seenPaths[referencePath]; duplicate { + return nil, fmt.Errorf("referenced file %q was selected more than once", reference.File) + } + seenPaths[referencePath] = struct{}{} + + referenceContent, err := syncreference.Render(reference.Format, resource.Variation) + if err != nil { + return nil, err + } + originalContent, err := os.ReadFile(referencePath) + if err != nil { + return nil, fmt.Errorf("read referenced file %q: %w", reference.File, err) + } + info, err := os.Stat(referencePath) + if err != nil { + return nil, fmt.Errorf("stat referenced file %q: %w", reference.File, err) + } + replacements = append(replacements, stagedVariation{ + relativePath: reference.File, + destinationPath: referencePath, + originalContent: originalContent, + replacementContent: referenceContent, + mode: info.Mode().Perm(), + }) + } + return replacements, nil +} + +// stageReplacements writes every replacement to its destination directory +// before the first original file is changed. +func stageReplacements(replacements []stagedVariation) error { + for index := range replacements { + stagedPath, err := stageReplacement(replacements[index]) + if err != nil { + removeStagedVariations(replacements) + return err + } + replacements[index].stagedPath = stagedPath + } + return nil +} + +// verifyReplacementSources detects editor changes made after preflight and +// before commit so sync never overwrites unreviewed content. +func verifyReplacementSources(replacements []stagedVariation) error { + for _, replacement := range replacements { + current, err := os.ReadFile(replacement.destinationPath) + if err != nil { + return fmt.Errorf("recheck variation %s: %w", replacement.relativePath, err) + } + if !bytes.Equal(current, replacement.originalContent) { + return fmt.Errorf("variation %s changed while syncing", replacement.relativePath) + } + } + return nil +} + +// commitReplacements renames staged files into place and restores already +// replaced files if a later rename fails. +func commitReplacements(replacements []stagedVariation) error { + var replaced []stagedVariation + for _, replacement := range replacements { + // Each rename is atomic, but the batch is not. Keep the committed prefix + // so it can be restored if a later destination fails. + if err := os.Rename(replacement.stagedPath, replacement.destinationPath); err != nil { + return errors.Join(fmt.Errorf("replace variation %s: %w", replacement.relativePath, err), rollbackVariations(replaced)) + } + replaced = append(replaced, replacement) + } + return nil +} + +// replacementPaths reports wrapper paths while hiding referenced-file details. +func replacementPaths(replacements []stagedVariation) []string { + var paths []string + for _, replacement := range replacements { + if replacement.report { + paths = append(paths, replacement.relativePath) + } + } + return paths +} + +// stageReplacement durably writes one temporary file beside its destination, +// preserving the destination's permission bits. +func stageReplacement(replacement stagedVariation) (string, error) { + temp, err := os.CreateTemp(filepath.Dir(replacement.destinationPath), "."+filepath.Base(replacement.destinationPath)+".tmp-") + if err != nil { + return "", fmt.Errorf("stage variation %s: %w", replacement.relativePath, err) + } + tempPath := temp.Name() + closeWithError := func(err error) (string, error) { + _ = temp.Close() + _ = os.Remove(tempPath) + return "", err + } + + if err := temp.Chmod(replacement.mode); err != nil { + return closeWithError(fmt.Errorf("set variation permissions %s: %w", replacement.relativePath, err)) + } + if _, err := temp.Write(replacement.replacementContent); err != nil { + return closeWithError(fmt.Errorf("write staged variation %s: %w", replacement.relativePath, err)) + } + if err := temp.Sync(); err != nil { + return closeWithError(fmt.Errorf("sync staged variation %s: %w", replacement.relativePath, err)) + } + if err := temp.Close(); err != nil { + _ = os.Remove(tempPath) + return "", fmt.Errorf("close staged variation %s: %w", replacement.relativePath, err) + } + return tempPath, nil +} + +// removeStagedVariations cleans up temporary files left after success or failure. +func removeStagedVariations(replacements []stagedVariation) { + for _, replacement := range replacements { + if replacement.stagedPath != "" { + _ = os.Remove(replacement.stagedPath) + } + } +} + +// rollbackVariations restores original bytes in reverse commit order. +func rollbackVariations(replacements []stagedVariation) error { + var rollbackErr error + // Reverse order mirrors the commit sequence and minimizes time spent in a + // partially restored state. + for index := len(replacements) - 1; index >= 0; index-- { + replacement := replacements[index] + tempPath, err := stageReplacement(stagedVariation{ + relativePath: replacement.relativePath, destinationPath: replacement.destinationPath, + replacementContent: replacement.originalContent, mode: replacement.mode, + }) + if err == nil { + err = os.Rename(tempPath, replacement.destinationPath) + if err != nil { + _ = os.Remove(tempPath) + } + } + if err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf("roll back variation %s: %w", replacement.relativePath, err)) + } + } + return rollbackErr +} diff --git a/internal/sync/local/store.go b/internal/sync/local/store.go new file mode 100644 index 00000000..4bdc75a1 --- /dev/null +++ b/internal/sync/local/store.go @@ -0,0 +1,251 @@ +package local + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "syscall" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +// ErrVariationExists reports that a create would overwrite a local wrapper. +var ErrVariationExists = errors.New("variation already exists locally") + +// VariationFile contains everything needed to write one local variation wrapper. +type VariationFile struct { + ProjectKey string + ConfigKey string + Upsert bool + Ref *Reference + Variation syncdomain.Variation +} + +// VariationReplacement identifies an existing wrapper and its replacement state. +type VariationReplacement struct { + ProjectKey string + ConfigKey string + Variation syncdomain.Variation +} + +// VariationDeletion identifies an existing wrapper to remove. +type VariationDeletion struct { + ProjectKey string + ConfigKey string + VariationKey string +} + +// RenderedVariationFile is a repository-relative wrapper ready to write. +type RenderedVariationFile struct { + Path string + Content []byte +} + +// Store reads and writes resources under a repository's .launchdarkly directory. +type Store struct { + repositoryRoot string + root string +} + +// NewStore creates a local resource store rooted at a Git repository. +func NewStore(repositoryRoot string) Store { + return Store{repositoryRoot: repositoryRoot, root: filepath.Join(repositoryRoot, syncdomain.RootDir)} +} + +// Exists reports whether the repository has a .launchdarkly directory. +func (store Store) Exists() (bool, error) { + info, err := os.Stat(store.root) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("inspect %s: %w", store.root, err) + } + if !info.IsDir() { + return false, fmt.Errorf("%s exists but is not a directory", store.root) + } + return true, nil +} + +// ProjectKeys returns locally managed project keys in deterministic order. +func (store Store) ProjectKeys() ([]string, error) { + entries, err := os.ReadDir(store.root) + if err != nil { + return nil, fmt.Errorf("read %s: %w", store.root, err) + } + + var keys []string + for _, entry := range entries { + if entry.IsDir() { + keys = append(keys, entry.Name()) + } + } + slices.Sort(keys) + return keys, nil +} + +// VariationExists reports whether one local variation wrapper exists. +func (store Store) VariationExists(projectKey, configKey, variationKey string) (bool, error) { + path, err := store.variationPath(projectKey, configKey, variationKey) + if err != nil { + return false, err + } + + _, err = os.Stat(path) + switch { + case err == nil: + return true, nil + case errors.Is(err, os.ErrNotExist): + return false, nil + default: + return false, fmt.Errorf("inspect variation %s: %w", variationKey, err) + } +} + +// Bootstrap atomically creates a new .launchdarkly directory. +func (store Store) Bootstrap(resources []VariationFile) ([]string, error) { + if _, err := os.Stat(store.root); err == nil { + return nil, fmt.Errorf("%s already exists", store.root) + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("inspect %s: %w", store.root, err) + } + + stagingDirectory, err := os.MkdirTemp(filepath.Dir(store.root), ".launchdarkly.tmp-") + if err != nil { + return nil, fmt.Errorf("create bootstrap staging directory: %w", err) + } + defer func() { _ = os.RemoveAll(stagingDirectory) }() + + // Build the entire workspace in a sibling directory. The final rename is a + // single commit point because source and destination share a filesystem. + stagedStore := Store{repositoryRoot: store.repositoryRoot, root: stagingDirectory} + paths, err := stagedStore.createVariations(resources) + if err != nil { + return nil, err + } + if err := os.Rename(stagingDirectory, store.root); err != nil { + return nil, fmt.Errorf("finish bootstrap: %w", err) + } + return paths, nil +} + +// Add creates a batch of variation wrappers without overwriting existing files. +func (store Store) Add(resources []VariationFile) ([]string, error) { + return store.createVariations(resources) +} + +type existingVariation struct { + relativePath string + absolutePath string + content []byte + mode os.FileMode + frontMatter variationFrontMatter +} + +// inspectVariation reads the wrapper metadata needed by replace and delete +// transactions. +func (store Store) inspectVariation(projectKey, configKey, variationKey string) (existingVariation, error) { + absolutePath, err := store.variationPath(projectKey, configKey, variationKey) + if err != nil { + return existingVariation{}, err + } + + content, err := os.ReadFile(absolutePath) + if err != nil { + return existingVariation{}, fmt.Errorf("read variation %s: %w", variationKey, err) + } + info, err := os.Stat(absolutePath) + if err != nil { + return existingVariation{}, fmt.Errorf("stat variation %s: %w", variationKey, err) + } + if !info.Mode().IsRegular() { + return existingVariation{}, fmt.Errorf("variation %s is not a regular file", variationKey) + } + + var frontMatter variationFrontMatter + if _, err := parseYAMLFrontMatter(content, &frontMatter); err != nil { + return existingVariation{}, fmt.Errorf("parse variation %s: %w", variationKey, err) + } + if err := validateVariation(filepath.Base(absolutePath), frontMatter); err != nil { + return existingVariation{}, fmt.Errorf("validate variation %s: %w", variationKey, err) + } + + return existingVariation{ + relativePath: filepath.ToSlash(strings.TrimPrefix(absolutePath, store.root+string(filepath.Separator))), + absolutePath: absolutePath, + content: content, + mode: info.Mode().Perm(), + frontMatter: frontMatter, + }, nil +} + +// RemoveEmptyDirectories removes empty resource directories left by deletes. +func (store Store) RemoveEmptyDirectories() error { + var directories []string + err := filepath.WalkDir(store.root, func(path string, entry os.DirEntry, walkErr error) error { + if errors.Is(walkErr, os.ErrNotExist) { + return nil + } + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + directories = append(directories, path) + } + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("inspect empty sync directories: %w", err) + } + + // Remove deepest-first so parent directories become empty as their children + // disappear. ENOTEMPTY is expected when a directory still owns resources. + for index := len(directories) - 1; index >= 0; index-- { + err := os.Remove(directories[index]) + if err == nil || errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTEMPTY) { + continue + } + return fmt.Errorf("remove empty sync directory %s: %w", directories[index], err) + } + return nil +} + +// variationPath validates every identity component before constructing a path +// beneath the managed workspace. +func (store Store) variationPath(projectKey, configKey, variationKey string) (string, error) { + segments := []struct { + name string + value string + }{ + {name: "project key", value: projectKey}, + {name: "config key", value: configKey}, + {name: "variation key", value: variationKey}, + } + for _, segment := range segments { + if err := validatePathSegment(segment.value); err != nil { + return "", fmt.Errorf("invalid %s %q: %w", segment.name, segment.value, err) + } + } + return filepath.Join(store.root, projectKey, configsDir, configKey, variationKey+variationFileSuffix), nil +} + +// validatePathSegment rejects traversal, separators, and null bytes before a +// resource identity reaches filesystem APIs. +func validatePathSegment(value string) error { + if value == "" { + return errors.New("must not be empty") + } + if value == "." || value == ".." || strings.ContainsAny(value, `/\`) { + return errors.New("must be a single path segment") + } + if strings.IndexByte(value, 0) >= 0 { + return errors.New("must not contain a null byte") + } + return nil +} diff --git a/internal/sync/local/store_test.go b/internal/sync/local/store_test.go new file mode 100644 index 00000000..97efa88a --- /dev/null +++ b/internal/sync/local/store_test.go @@ -0,0 +1,424 @@ +package local + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestStore_ProjectKeys(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + require.NoError(t, os.MkdirAll( + filepath.Join(root, syncdomain.RootDir, "zeta"), + 0o755, + )) + require.NoError(t, os.MkdirAll( + filepath.Join(root, syncdomain.RootDir, "alpha"), + 0o755, + )) + require.NoError(t, os.WriteFile( + filepath.Join(root, syncdomain.RootDir, "README"), + nil, + 0o644, + )) + + keys, err := store.ProjectKeys() + + require.NoError(t, err) + assert.Equal(t, []string{"alpha", "zeta"}, keys) +} + +func TestStore_BootstrapRoundTripsSupportedModes(t *testing.T) { + root := t.TempDir() + resources := []VariationFile{ + { + ProjectKey: "project", + ConfigKey: "completion-config", + Upsert: true, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeCompletion, + Key: "friendly", + Name: "Friendly", + ModelConfigKey: "claude", + ModelConfigVersion: 3, + Model: map[string]any{"modelName": "claude"}, + OutputFormat: map[string]any{"type": "json_schema"}, + Messages: []syncdomain.Message{ + {Role: "system", Content: `Explain nested tags from C:\prompts.`}, + {Role: "user", Content: "Answer the question."}, + }, + }, + }, + { + ProjectKey: "project", + ConfigKey: "agent-config", + Upsert: true, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeAgent, + Key: "researcher", + Name: "Researcher", + Instructions: "Check the available sources.", + }, + }, + } + + paths, err := NewStore(root).Bootstrap(resources) + require.NoError(t, err) + assert.ElementsMatch(t, []string{ + "project/configs/completion-config/friendly.prompt.md", + "project/configs/agent-config/researcher.prompt.md", + }, paths) + + agentFile, err := os.ReadFile(filepath.Join( + root, + syncdomain.RootDir, + "project", + configsDir, + "agent-config", + "researcher.prompt.md", + )) + require.NoError(t, err) + assert.NotContains(t, string(agentFile), "instructions:") + assert.Contains(t, string(agentFile), "\nCheck the available sources.\n") + + compiled, err := Compile(os.DirFS(root)) + require.NoError(t, err) + require.Len(t, compiled, len(resources)) + for _, local := range resources { + resource := requireVariationResource( + t, + compiled, + local.ConfigKey+"/"+local.Variation.Key, + ) + expected, err := marshalPayload(local.Variation) + require.NoError(t, err) + assert.JSONEq(t, string(expected), string(resource.Payload)) + assert.True(t, resource.Upsert) + } +} + +func TestStore_BootstrapNormalizesPromptLineEndings(t *testing.T) { + tests := map[string]syncdomain.Variation{ + "agent": { + Mode: syncdomain.VariationModeAgent, Key: "agent", Name: "Agent", + Instructions: "First line.\r\nSecond line.\rThird line.", + }, + "completion": { + Mode: syncdomain.VariationModeCompletion, Key: "completion", Name: "Completion", + Messages: []syncdomain.Message{{Role: "system", Content: "First line.\r\nSecond line.\rThird line."}}, + }, + } + + for name, serverVariation := range tests { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + _, err := NewStore(root).Bootstrap([]VariationFile{{ + ProjectKey: "project", ConfigKey: "config", Variation: serverVariation, + }}) + require.NoError(t, err) + + wrapperPath := filepath.Join(root, syncdomain.RootDir, "project", configsDir, "config", serverVariation.Key+variationFileSuffix) + wrapper, err := os.ReadFile(wrapperPath) + require.NoError(t, err) + require.NotContains(t, string(wrapper), "\r") + + compiled, err := Compile(os.DirFS(root)) + require.NoError(t, err) + resource := requireVariationResource(t, compiled, "config/"+serverVariation.Key) + var localVariation syncdomain.Variation + require.NoError(t, json.Unmarshal(resource.Payload, &localVariation)) + + serverFingerprint, err := syncdomain.FingerprintVariation("project", "config/"+serverVariation.Key, serverVariation) + require.NoError(t, err) + localFingerprint, err := syncdomain.FingerprintVariation("project", "config/"+serverVariation.Key, localVariation) + require.NoError(t, err) + require.Equal(t, serverFingerprint, localFingerprint) + }) + } +} + +func TestStore_RenderVariationsMatchesWrittenFile(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + resource := localVariation("preview") + + rendered, err := store.RenderVariations([]VariationFile{resource}) + + require.NoError(t, err) + require.Len(t, rendered, 1) + assert.Equal(t, "project/configs/config/preview.prompt.md", rendered[0].Path) + _, err = os.Stat(filepath.Join(root, syncdomain.RootDir)) + assert.ErrorIs(t, err, os.ErrNotExist) + + paths, err := store.Bootstrap([]VariationFile{resource}) + require.NoError(t, err) + assert.Equal(t, []string{rendered[0].Path}, paths) + content, err := os.ReadFile(filepath.Join( + root, + syncdomain.RootDir, + filepath.FromSlash(rendered[0].Path), + )) + require.NoError(t, err) + assert.Equal(t, rendered[0].Content, content) +} + +func TestStore_AddNeverOverwritesExistingVariation(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + existing := VariationFile{ + ProjectKey: "project", + ConfigKey: "config", + Upsert: true, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeCompletion, + Key: "existing", + Name: "Existing", + }, + } + _, err := store.Add([]VariationFile{existing}) + require.NoError(t, err) + + path := filepath.Join( + root, + syncdomain.RootDir, + "project", + configsDir, + "config", + "existing.prompt.md", + ) + before, err := os.ReadFile(path) + require.NoError(t, err) + + existing.Variation.Name = "Changed on server" + _, err = store.Add([]VariationFile{existing}) + require.ErrorIs(t, err, ErrVariationExists) + + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, before, after) +} + +func TestStore_AddRollsBackNewFilesWhenOneExists(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + existing := localVariation("existing") + _, err := store.Add([]VariationFile{existing}) + require.NoError(t, err) + + newVariation := localVariation("new") + newVariation.ProjectKey = "new-project" + _, err = store.Add([]VariationFile{newVariation, existing}) + require.ErrorIs(t, err, ErrVariationExists) + + exists, err := store.VariationExists("new-project", "config", "new") + require.NoError(t, err) + assert.False(t, exists) + _, statErr := os.Stat(filepath.Join(root, syncdomain.RootDir, "new-project")) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestStore_AddValidationFailureLeavesNoWorkspace(t *testing.T) { + root := t.TempDir() + resource := localVariation("../unsafe") + + _, err := NewStore(root).Add([]VariationFile{resource}) + + require.ErrorContains(t, err, "must be a single path segment") + _, statErr := os.Stat(filepath.Join(root, syncdomain.RootDir)) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestStore_ReplaceVariationsPreservesLocalMetadata(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + existing := localVariation("existing") + existing.Upsert = false + _, err := store.Add([]VariationFile{existing}) + require.NoError(t, err) + + path := filepath.Join( + root, + syncdomain.RootDir, + "project", + configsDir, + "config", + "existing.prompt.md", + ) + require.NoError(t, os.Chmod(path, 0o640)) + + server := existing.Variation + server.Name = "Changed on server" + server.ModelConfigKey = "claude" + server.ModelConfigVersion = 3 + server.Messages = []syncdomain.Message{{ + Role: "system", + Content: "Use the server instructions.", + }} + paths, err := store.ReplaceVariations([]VariationReplacement{{ + ProjectKey: "project", + ConfigKey: "config", + Variation: server, + }}) + + require.NoError(t, err) + assert.Equal(t, []string{"project/configs/config/existing.prompt.md"}, paths) + + compiled, err := Compile(os.DirFS(root)) + require.NoError(t, err) + resource := requireVariationResource(t, compiled, "config/existing") + assert.False(t, resource.Upsert) + expected, err := marshalPayload(server) + require.NoError(t, err) + assert.JSONEq(t, string(expected), string(resource.Payload)) + + info, err := os.Stat(path) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o640), info.Mode().Perm()) +} + +func TestStore_ReplaceVariationsPreflightsBeforeWriting(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + existing := localVariation("existing") + _, err := store.Add([]VariationFile{existing}) + require.NoError(t, err) + + path := filepath.Join( + root, + syncdomain.RootDir, + "project", + configsDir, + "config", + "existing.prompt.md", + ) + before, err := os.ReadFile(path) + require.NoError(t, err) + + changed := existing.Variation + changed.Name = "Changed on server" + _, err = store.ReplaceVariations([]VariationReplacement{ + { + ProjectKey: "project", + ConfigKey: "config", + Variation: changed, + }, + { + ProjectKey: "project", + ConfigKey: "config", + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeCompletion, + Key: "missing", + Name: "Missing", + }, + }, + }) + + require.ErrorIs(t, err, os.ErrNotExist) + after, readErr := os.ReadFile(path) + require.NoError(t, readErr) + assert.Equal(t, before, after) +} + +func TestStore_DeleteVariationsRemovesFileAndEmptyWorkspace(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + _, err := store.Add([]VariationFile{localVariation("existing")}) + require.NoError(t, err) + + paths, err := store.DeleteVariations([]VariationDeletion{{ + ProjectKey: "project", + ConfigKey: "config", + VariationKey: "existing", + }}) + + require.NoError(t, err) + assert.Equal(t, []string{"project/configs/config/existing.prompt.md"}, paths) + _, statErr := os.Stat(filepath.Join(root, syncdomain.RootDir)) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestStore_DeleteVariationsPreflightsBeforeDeleting(t *testing.T) { + root := t.TempDir() + store := NewStore(root) + _, err := store.Add([]VariationFile{localVariation("existing")}) + require.NoError(t, err) + + _, err = store.DeleteVariations([]VariationDeletion{ + {ProjectKey: "project", ConfigKey: "config", VariationKey: "existing"}, + {ProjectKey: "project", ConfigKey: "config", VariationKey: "missing"}, + }) + + require.ErrorIs(t, err, os.ErrNotExist) + exists, existsErr := store.VariationExists("project", "config", "existing") + require.NoError(t, existsErr) + assert.True(t, exists) +} + +func TestStore_BootstrapFailureLeavesNoDirectory(t *testing.T) { + root := t.TempDir() + resource := localVariation("../unsafe") + + _, err := NewStore(root).Bootstrap([]VariationFile{resource}) + require.ErrorContains(t, err, "must be a single path segment") + _, statErr := os.Stat(filepath.Join(root, syncdomain.RootDir)) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestStore_RejectsUnsupportedMessageRole(t *testing.T) { + root := t.TempDir() + resource := localVariation("unsupported-role") + resource.Variation.Messages = []syncdomain.Message{{Role: "developer", Content: "result"}} + + _, err := NewStore(root).Bootstrap([]VariationFile{resource}) + require.ErrorContains(t, err, `unsupported message role "developer"`) + _, statErr := os.Stat(filepath.Join(root, syncdomain.RootDir)) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestStore_BootstrapRefusesExistingDirectory(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, syncdomain.RootDir), 0o755)) + + _, err := NewStore(root).Bootstrap([]VariationFile{localVariation("new")}) + require.Error(t, err) + assert.False(t, errors.Is(err, os.ErrNotExist)) +} + +func localVariation(key string) VariationFile { + return VariationFile{ + ProjectKey: "project", + ConfigKey: "config", + Upsert: true, + Variation: syncdomain.Variation{ + Mode: syncdomain.VariationModeCompletion, + Key: key, + Name: key, + }, + } +} + +func requireVariationResource( + t *testing.T, + resources []syncdomain.SyncedResource, + lookupKey string, +) syncdomain.SyncedResource { + t.Helper() + + for _, resource := range resources { + if resource.Kind == syncdomain.KindVariation && resource.LookupKey == lookupKey { + return resource + } + } + + require.FailNow(t, "variation resource not found", lookupKey) + + return syncdomain.SyncedResource{} +} diff --git a/internal/sync/local/variation.go b/internal/sync/local/variation.go new file mode 100644 index 00000000..e8f11224 --- /dev/null +++ b/internal/sync/local/variation.go @@ -0,0 +1,291 @@ +package local + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + + "github.com/adrg/frontmatter" + syncdomain "github.com/launchdarkly/ldcli/internal/sync" + syncreference "github.com/launchdarkly/ldcli/internal/sync/reference" + "gopkg.in/yaml.v3" +) + +const ( + configsDir = "configs" + variationFileSuffix = ".prompt.md" +) + +type localFile struct { + ProjectKey string + RelPath string + Data []byte + ReadReference func(Reference) ([]byte, error) +} + +type variationFrontMatter struct { + FormatVersion int `yaml:"formatVersion"` + Upsert bool `yaml:"upsert"` + Ref *Reference `yaml:"ref,omitempty"` + syncdomain.Variation `yaml:",inline"` +} + +var yamlFrontMatter = frontmatter.NewFormat("---", "---", func(data []byte, destination any) error { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + + return decoder.Decode(destination) +}) + +// isVariationFile recognizes direct children of a config directory with the +// supported wrapper suffix. +func isVariationFile(relPath string) bool { + if !strings.HasSuffix(relPath, variationFileSuffix) { + return false + } + + dir, file := path.Split(relPath) + dir = strings.TrimSuffix(dir, "/") + + return dir != "" && !strings.Contains(dir, "/") && file != "" +} + +// parseVariation combines wrapper metadata with inline or referenced prompt +// content and emits the canonical resource payload. +func parseVariation(file localFile) (syncdomain.SyncedResource, error) { + var meta variationFrontMatter + body, err := parseYAMLFrontMatter(file.Data, &meta) + if err != nil { + return syncdomain.SyncedResource{}, err + } + + if err := validateVariation(file.RelPath, meta); err != nil { + return syncdomain.SyncedResource{}, err + } + + variation := meta.Variation + if meta.Ref != nil { + // The wrapper owns identity and model metadata; the adapter supplies + // only the content fields represented by the external source format. + if strings.TrimSpace(string(body)) != "" { + return syncdomain.SyncedResource{}, errors.New("referenced variation cannot also contain an inline prompt body") + } + referencedContent, err := file.ReadReference(*meta.Ref) + if err != nil { + return syncdomain.SyncedResource{}, err + } + if _, err := syncreference.ApplyToVariation(meta.Ref.Format, referencedContent, &variation); err != nil { + return syncdomain.SyncedResource{}, err + } + stem := strings.TrimSuffix(path.Base(file.RelPath), variationFileSuffix) + if variation.Key != stem { + return syncdomain.SyncedResource{}, fmt.Errorf("referenced prompt key %q does not match filename %q", variation.Key, stem) + } + } else { + // Inline bodies use the simplest representation for each mode: plain + // instructions for agents and explicit role blocks for completions. + switch variation.Mode { + case syncdomain.VariationModeAgent: + variation.Instructions = syncdomain.NormalizePromptText(string(body)) + case syncdomain.VariationModeCompletion: + messages, err := parseCompletionMessages(string(body)) + if err != nil { + return syncdomain.SyncedResource{}, err + } + variation.Messages = messages + } + } + + payload, err := marshalPayload(variation) + if err != nil { + return syncdomain.SyncedResource{}, err + } + + configKey := path.Dir(file.RelPath) + + return syncdomain.SyncedResource{ + Kind: syncdomain.KindVariation, + ProjectKey: file.ProjectKey, + LookupKey: configKey + "/" + meta.Key, + Payload: payload, + Upsert: meta.Upsert, + }, nil +} + +// validateVariation checks the file format and binds the declared key to the filename. +func validateVariation(relPath string, meta variationFrontMatter) error { + switch { + case meta.FormatVersion == 0: + return errors.New("formatVersion is required") + case meta.FormatVersion != 1: + return fmt.Errorf("unsupported formatVersion %d", meta.FormatVersion) + case meta.Mode == "": + return errors.New("mode is required") + case !meta.Mode.Valid(): + return fmt.Errorf("unsupported mode %q", meta.Mode) + case meta.Key == "": + return errors.New("key is required") + case meta.Name == "": + return errors.New("name is required") + } + + stem := strings.TrimSuffix(path.Base(relPath), variationFileSuffix) + if stem != meta.Key { + return fmt.Errorf("key %q does not match filename %q", meta.Key, stem) + } + + return nil +} + +// marshalPayload encodes canonical JSON without HTML escaping or a trailing newline. +func marshalPayload(value any) (json.RawMessage, error) { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + + if err := encoder.Encode(value); err != nil { + return nil, fmt.Errorf("marshal payload: %w", err) + } + + return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil +} + +// parseYAMLFrontMatter accepts BOM and common newline variants, decodes strict +// YAML metadata, and returns the remaining prompt body. +func parseYAMLFrontMatter(data []byte, destination any) ([]byte, error) { + source := bytes.TrimLeft(bytes.TrimPrefix(data, []byte("\ufeff")), "\r\n") + hasStart := bytes.Equal(source, []byte("---")) || + bytes.HasPrefix(source, []byte("---\n")) || + bytes.HasPrefix(source, []byte("---\r\n")) + if !hasStart { + return nil, errors.New("missing YAML front matter") + } + + body, err := frontmatter.MustParse(bytes.NewReader(source), destination, yamlFrontMatter) + if errors.Is(err, frontmatter.ErrNotFound) { + return nil, errors.New("unclosed YAML front matter") + } + if err != nil { + return nil, fmt.Errorf("invalid front matter: %w", err) + } + + return body, nil +} + +var messageRoles = []string{"system", "user", "assistant"} + +// parseCompletionMessages parses tagged role blocks, with untagged text treated +// as one system message for a simple authoring experience. +func parseCompletionMessages(body string) ([]syncdomain.Message, error) { + body = syncdomain.NormalizePromptText(body) + if body == "" { + return nil, nil + } + + if _, _, _, ok := nextOpenTag(body, 0); !ok { + // A plain body is a convenient shorthand for the common single-system + // message case. + return []syncdomain.Message{{ + Role: "system", + Content: body, + }}, nil + } + + var messages []syncdomain.Message + cursor := 0 + + for cursor < len(body) { + start, role, contentStart, ok := nextOpenTag(body, cursor) + if !ok { + if strings.TrimSpace(body[cursor:]) != "" { + return nil, errors.New("unexpected text outside message tags") + } + + break + } + if strings.TrimSpace(body[cursor:start]) != "" { + return nil, errors.New("unexpected text outside message tags") + } + + contentEnd, closeEnd, found := matchingClose(body, contentStart, role) + if !found { + return nil, fmt.Errorf("unclosed <%s> tag", role) + } + + // Advance to the byte immediately after the balanced closing tag. The + // next iteration verifies that only whitespace separates messages. + messages = append(messages, syncdomain.Message{ + Role: role, + Content: unescapeMessageContent(syncdomain.NormalizePromptText(body[contentStart:contentEnd]), role), + }) + cursor = closeEnd + } + + return messages, nil +} + +// unescapeMessageContent reverses the delimiter escaping applied during rendering. +func unescapeMessageContent(content, role string) string { + content = strings.ReplaceAll(content, `<\/`+role+">", "") + content = strings.ReplaceAll(content, `<\`+role+">", "<"+role+">") + return strings.ReplaceAll(content, `\\`, `\`) +} + +// nextOpenTag finds the earliest supported role tag at or after an offset. +func nextOpenTag(body string, from int) (start int, role string, contentStart int, ok bool) { + start = -1 + + for _, candidate := range messageRoles { + tag := "<" + candidate + ">" + index := strings.Index(body[from:], tag) + if index < 0 { + continue + } + + absolute := from + index + if start < 0 || absolute < start { + start = absolute + role = candidate + contentStart = absolute + len(tag) + ok = true + } + } + + return start, role, contentStart, ok +} + +// matchingClose finds the balanced closing tag for one role block. +func matchingClose(body string, from int, role string) (contentEnd, closeEnd int, ok bool) { + open := "<" + role + ">" + close := "" + depth := 1 + index := from + + for index < len(body) { + relativeOpen := strings.Index(body[index:], open) + relativeClose := strings.Index(body[index:], close) + if relativeClose < 0 { + return 0, 0, false + } + + if relativeOpen >= 0 && relativeOpen < relativeClose { + depth++ + index += relativeOpen + len(open) + + continue + } + + depth-- + closeAt := index + relativeClose + if depth == 0 { + return closeAt, closeAt + len(close), true + } + + index = closeAt + len(close) + } + + return 0, 0, false +} diff --git a/internal/sync/local/variation_test.go b/internal/sync/local/variation_test.go new file mode 100644 index 00000000..6b8f949e --- /dev/null +++ b/internal/sync/local/variation_test.go @@ -0,0 +1,256 @@ +package local + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + syncdomain "github.com/launchdarkly/ldcli/internal/sync" +) + +func TestIsVariationFile(t *testing.T) { + assert.True(t, isVariationFile("my-config/my-variation.prompt.md")) + assert.False(t, isVariationFile("my-variation.prompt.md")) + assert.False(t, isVariationFile("my-config/nested/my-variation.prompt.md")) + assert.False(t, isVariationFile("my-config/my-variation.prompt")) + assert.False(t, isVariationFile("my-config/my-variation.md")) +} + +func TestParseVariation_UntaggedBodyIsSystemMessage(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/plain.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: completion +key: plain +name: Plain +--- + +Just say hello. +`), + } + + resource, err := parseVariation(file) + require.NoError(t, err) + + var payload syncdomain.Variation + require.NoError(t, unmarshalPayload(resource, &payload)) + require.Equal( + t, + []syncdomain.Message{{Role: "system", Content: "Just say hello."}}, + payload.Messages, + ) +} + +func TestParseVariation_AgentBodyIsInstructions(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/agent.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: agent +key: agent +name: Agent +--- + +Use the available capabilities. + +This tag is part of the instructions. +`), + } + + resource, err := parseVariation(file) + require.NoError(t, err) + + var payload syncdomain.Variation + require.NoError(t, unmarshalPayload(resource, &payload)) + assert.Equal( + t, + "Use the available capabilities.\n\nThis tag is part of the instructions.", + payload.Instructions, + ) + assert.Empty(t, payload.Messages) +} + +func TestParseVariation_ParsesBOMAndCRLF(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/plain.prompt.md", + Data: []byte("\ufeff\r\n---\r\n" + + "formatVersion: 1\r\n" + + "mode: completion\r\n" + + "key: plain\r\n" + + "name: Plain\r\n" + + "---\r\n\r\n" + + "Just say hello.\r\n"), + } + + resource, err := parseVariation(file) + require.NoError(t, err) + + var payload syncdomain.Variation + require.NoError(t, unmarshalPayload(resource, &payload)) + require.Equal( + t, + []syncdomain.Message{{Role: "system", Content: "Just say hello."}}, + payload.Messages, + ) +} + +func TestParseVariation_RejectsBodyFieldsInFrontMatter(t *testing.T) { + for _, field := range []string{ + "instructions: Use the available capabilities.", + "messages: []", + } { + t.Run(field, func(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/agent.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: agent +key: agent +name: Agent +` + field + ` +--- +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "invalid front matter") + }) + } +} + +func TestParseVariation_RejectsUnclosedFrontMatter(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/v.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: completion +key: v +name: V +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "unclosed YAML front matter") +} + +func TestParseVariation_MismatchedTags(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/bad.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: completion +key: bad +name: Bad +--- + + +oops + +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "unclosed tag") +} + +func TestParseVariation_TextOutsideTags(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/bad.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: completion +key: bad +name: Bad +--- + +hello + +hi + +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "unexpected text outside message tags") +} + +func TestParseVariation_RequiresFormatVersion(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/v.prompt.md", + Data: []byte(`--- +mode: completion +key: v +name: V +--- +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "formatVersion is required") +} + +func TestParseVariation_RequiresMode(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/v.prompt.md", + Data: []byte(`--- +formatVersion: 1 +key: v +name: V +--- +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "mode is required") +} + +func TestParseVariation_RejectsUnsupportedMode(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/v.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: other +key: v +name: V +--- +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, `unsupported mode "other"`) +} + +func TestParseVariation_RejectsVariationDescription(t *testing.T) { + file := localFile{ + ProjectKey: "proj", + RelPath: "cfg/v.prompt.md", + Data: []byte(`--- +formatVersion: 1 +mode: completion +key: v +name: V +description: Variation description +--- +`), + } + + _, err := parseVariation(file) + require.ErrorContains(t, err, "invalid front matter") +} + +func unmarshalPayload(resource syncdomain.SyncedResource, destination any) error { + return json.Unmarshal(resource.Payload, destination) +} diff --git a/internal/sync/reference/adapters/plain_markdown/plain_markdown.go b/internal/sync/reference/adapters/plain_markdown/plain_markdown.go index 5f5830d6..9b273561 100644 --- a/internal/sync/reference/adapters/plain_markdown/plain_markdown.go +++ b/internal/sync/reference/adapters/plain_markdown/plain_markdown.go @@ -2,8 +2,8 @@ package plain_markdown import ( "fmt" - "strings" + syncdomain "github.com/launchdarkly/ldcli/internal/sync" "github.com/launchdarkly/ldcli/internal/sync/reference/adapters" ) @@ -13,7 +13,7 @@ type Adapter struct{} // Parse reads raw text as one system message. Raw files do not provide mode, // key, or name metadata. func (Adapter) Parse(content []byte) (adapters.Prompt, error) { - body := normalizeContent(string(content)) + body := syncdomain.NormalizePromptText(string(content)) prompt := adapters.Prompt{} if body != "" { prompt.Messages = []adapters.Message{{Role: adapters.RoleSystem, Content: body}} @@ -29,19 +29,11 @@ func (Adapter) Render(prompt adapters.Prompt) ([]byte, error) { if len(prompt.Messages) == 0 { return nil, nil } - body := normalizeContent(prompt.Messages[0].Content) + body := syncdomain.NormalizePromptText(prompt.Messages[0].Content) if body == "" { return nil, nil } return []byte(body + "\n"), nil } -// normalizeContent gives equivalent text files one platform-independent -// representation before they are fingerprinted or written. -func normalizeContent(content string) string { - content = strings.ReplaceAll(content, "\r\n", "\n") - content = strings.ReplaceAll(content, "\r", "\n") - return strings.TrimSpace(content) -} - var _ adapters.Adapter = Adapter{} diff --git a/internal/sync/resource.go b/internal/sync/resource.go index d4ee9e74..638d50f3 100644 --- a/internal/sync/resource.go +++ b/internal/sync/resource.go @@ -66,6 +66,14 @@ type Message struct { Content string `json:"content"` } +// NormalizePromptText gives semantically equivalent prompt text one stable +// representation across API responses, local files, and operating systems. +func NormalizePromptText(content string) string { + content = strings.ReplaceAll(content, "\r\n", "\n") + content = strings.ReplaceAll(content, "\r", "\n") + return strings.TrimSpace(content) +} + // Variation is the common prompt variation representation used by sync. type Variation struct { Mode VariationMode `json:"mode" yaml:"mode"`