From 78cc535d7b213214d596a89fdf253d0c72f23633 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 19:48:45 -0400 Subject: [PATCH 01/10] feat: add locale and ICU correctness core Canonicalize locale identity across configuration, state, translation memory, and policy hashes. Parse and structurally validate ICU messages before provider or translation-memory output is persisted. Signed-off-by: Tom Main --- README.md | 24 +- THIRD_PARTY_NOTICES.md | 32 + cmd/internationalizer/translate.go | 16 +- docs/localization-core-v2.md | 49 ++ go.mod | 3 +- go.sum | 2 + internal/config/config.go | 83 +- internal/config/config_test.go | 80 ++ internal/locale/locale.go | 209 +++++ internal/locale/locale_test.go | 125 +++ internal/message/message.go | 748 ++++++++++++++++++ internal/message/message_test.go | 222 ++++++ .../fuzz/FuzzParsePrint/87dbf68552061a26 | 2 + .../fuzz/FuzzParsePrint/b429763ebca9b738 | 2 + internal/policy/policy.go | 22 +- internal/policy/policy_test.go | 19 + internal/state/manifest.go | 23 +- internal/state/manifest_test.go | 44 ++ internal/tm/tm.go | 22 +- internal/tm/tm_test.go | 27 + internal/translate/protected.go | 5 +- internal/translate/protected_test.go | 4 +- internal/translate/translate.go | 39 +- internal/translate/translate_test.go | 109 +++ internal/validate/findings.go | 3 + internal/validate/plurals.go | 82 +- internal/validate/strict_test.go | 12 +- internal/validate/validate.go | 42 + internal/validate/validate_test.go | 97 +++ 29 files changed, 2024 insertions(+), 123 deletions(-) create mode 100644 THIRD_PARTY_NOTICES.md create mode 100644 docs/localization-core-v2.md create mode 100644 internal/locale/locale.go create mode 100644 internal/locale/locale_test.go create mode 100644 internal/message/message.go create mode 100644 internal/message/message_test.go create mode 100644 internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26 create mode 100644 internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738 diff --git a/README.md b/README.md index 6f7009d..179dbf7 100644 --- a/README.md +++ b/README.md @@ -138,8 +138,9 @@ baseline. ### `validate` Check all locale files against their source bundles. Default validation checks -structural coverage (the percentage of required target keys present), reports extra keys -as warnings, and fails for missing keys or interpolation mismatches. +structural coverage (the percentage of required target keys present), reports +extra keys as warnings, and fails for missing keys, interpolation mismatches, +or invalid ICU MessageFormat structure. ```bash internationalizer validate # human-readable output @@ -170,6 +171,9 @@ Human and JSON reports use stable finding codes: | `protected_structure_mismatch` | Interpolation, HTML, code, or link structure changed | | `glossary_violation` | No approved target term or variant was found | | `plural_form_missing` | A configured locale plural form is absent | +| `icu_message_syntax` | A source or target ICU message is malformed | +| `icu_argument_mismatch` | ICU argument names, types, or formatter styles differ | +| `icu_selector_mismatch` | Selectors differ or a plural category is invalid for the target locale | | `untracked` | No manifest record exists for the target | | `source_stale` | Source content changed after the recorded translation | | `policy_stale` | The generated prompt or model settings changed | @@ -299,6 +303,18 @@ validation: plural_style: i18next-v4 # generate and validate target-locale plural forms ``` +Locale identifiers must be well-formed BCP 47 tags such as `fr`, `pt-BR`, or +`sr-Latn-RS`. Canonical-equivalent target locales are rejected as duplicates, +and locale-specific provider overrides match canonical-equivalent spelling. + +ICU MessageFormat values are parsed structurally. Simple arguments, `select`, +`plural`, `selectordinal`, `number`, `date`, and `time` are supported, including +nested messages, plural offsets, exact-number selectors, and `#`. Validation +checks syntax, argument types and formatter styles, plural offsets, select +branch identity, and target-locale CLDR plural categories. Provider output that +breaks these invariants is rejected before a locale file or translation-memory +record is written. + With `i18next-v4`, recognized source plural families are expanded during translation to the target locale's CLDR categories. A target-only category uses the source family's `_other` value as its translation template. Strict @@ -392,6 +408,8 @@ internal/ openai.go OpenAI / compatible backend gemini.go Google Gemini via AI Studio backend OpenRouter uses openai.go with custom base_url + locale/ BCP 47 identity and CLDR plural categories + message/ ICU MessageFormat parser and structural comparison styleguide/ Style guide loader tm/ JSONL translation memory translate/ Translation orchestrator @@ -415,6 +433,8 @@ internal/ [AGPL-3.0](LICENSE) +See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for dependency notices. + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines. All contributions require DCO sign-off. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..e6e82e0 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,32 @@ +# Third-party notices + +Internationalizer includes software developed by third parties. Their licenses +apply to those components; the Internationalizer source remains licensed under +AGPL-3.0. + +## golang.org/x/text + +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +- Neither the name of Google LLC nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cmd/internationalizer/translate.go b/cmd/internationalizer/translate.go index 1ab9fc0..2fa232e 100644 --- a/cmd/internationalizer/translate.go +++ b/cmd/internationalizer/translate.go @@ -48,8 +48,12 @@ func newTranslateCmd() *cobra.Command { selectedLocales = cfg.TargetLocales } needsDefaultProvider := false - for _, locale := range selectedLocales { - if _, ok := cfg.LLM.LocaleOverrides[locale]; !ok { + for _, requested := range selectedLocales { + locale, ok := cfg.ConfiguredTargetLocale(requested) + if !ok { + return fmt.Errorf("locale %q is not in target_locales", requested) + } + if !cfg.HasLLMOverrideForLocale(locale) { needsDefaultProvider = true break } @@ -60,8 +64,12 @@ func newTranslateCmd() *cobra.Command { return err } } - for _, locale := range selectedLocales { - if _, ok := cfg.LLM.LocaleOverrides[locale]; !ok { + for _, requested := range selectedLocales { + locale, ok := cfg.ConfiguredTargetLocale(requested) + if !ok { + return fmt.Errorf("locale %q is not in target_locales", requested) + } + if !cfg.HasLLMOverrideForLocale(locale) { continue } localeLLM := cfg.LLMForLocale(locale) diff --git a/docs/localization-core-v2.md b/docs/localization-core-v2.md new file mode 100644 index 0000000..a300375 --- /dev/null +++ b/docs/localization-core-v2.md @@ -0,0 +1,49 @@ +# Localization Core v2 contract + +Internationalizer remains a local-first translation compiler for applications +that own their runtime. This contract does not add a hosted review service, +framework runtime, translation CDN, or source-rewriting setup command. + +## Locale identity + +- Configuration accepts well-formed BCP 47 language tags and preserves their + spelling in file paths. +- Equality, provider overrides, validation, and state identity use canonical + BCP 47 form. Canonical-equivalent target locales are duplicates. +- Private-use language tags remain valid. Path separators and malformed tags + are rejected before resolving target paths. + +## Message semantics + +- ICU messages are parsed structurally rather than treated as brace-shaped + interpolation strings. +- Supported arguments are simple interpolation, `select`, `plural`, + `selectordinal`, `number`, `date`, and `time`. +- Selectors may contain nested messages. Plural arguments support exact-number + selectors, offsets, and `#`. +- Every `select`, `plural`, and `selectordinal` requires `other`. +- A target must retain source argument names, argument kinds, formatter styles, + and plural offsets. Select branch identities are semantic and must be + preserved. Target plurals may add exact selectors and locale-applicable CLDR + categories. +- Parsing and printing are deterministic so later source-unit hashing can use + canonical message structure. + +## Review transfer + +Manifest v2 will represent generated, needs-review, approved, and manual +origins explicitly. Only an exact source-and-policy match may inherit approval. +Fuzzy candidates, changed policy, or changed message structure require review. + +## Source units + +Structured documents and source adapters will identify a unit by adapter, +normalized project-relative path, semantic location, and source structure. +Formatting offsets alone are not stable identity. + +## Compatibility boundary + +Existing JSON/YAML key workflows and i18next-v4 plural suffixes remain +supported. Canonical locale comparison may reject configurations that currently +spell the same locale more than once. ICU structural failures are deterministic +validation errors and block provider output from being written. diff --git a/go.mod b/go.mod index 3fd2061..ba5e1b7 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,10 @@ module github.com/Tom-R-Main/Internationalizer -go 1.25 +go 1.25.0 require ( github.com/spf13/cobra v1.10.2 + golang.org/x/text v0.36.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index eaa9638..1053042 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/config/config.go b/internal/config/config.go index d2107e5..4efac7f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,9 +4,10 @@ import ( "fmt" "os" "path/filepath" - "regexp" + "sort" "strings" + localeid "github.com/Tom-R-Main/Internationalizer/internal/locale" "gopkg.in/yaml.v3" ) @@ -133,7 +134,7 @@ func (c *Config) LLMForLocale(locale string) LLM { global.LocaleOverrides = nil applyLLMDefaults(&global) - override, ok := c.LLM.LocaleOverrides[locale] + override, ok := localeOverride(c.LLM.LocaleOverrides, locale) if !ok { return global } @@ -165,6 +166,29 @@ func (c *Config) LLMForLocale(locale string) LLM { return effective } +// ConfiguredTargetLocale returns the configured spelling of a target locale +// that is canonically equivalent to requested. +func (c *Config) ConfiguredTargetLocale(requested string) (string, bool) { + canonical, err := localeid.Canonical(requested) + if err != nil { + return "", false + } + for _, candidate := range c.TargetLocales { + candidateCanonical, candidateErr := localeid.Canonical(candidate) + if candidateErr == nil && candidateCanonical == canonical { + return candidate, true + } + } + return "", false +} + +// HasLLMOverrideForLocale reports whether a canonical-equivalent locale has +// explicit provider settings. +func (c *Config) HasLLMOverrideForLocale(locale string) bool { + _, ok := localeOverride(c.LLM.LocaleOverrides, locale) + return ok +} + func (c *Config) Validate() error { if err := c.ValidateProject(); err != nil { return err @@ -177,24 +201,38 @@ func (c *Config) ValidateProject() error { if c.Validation.PluralStyle != "" && c.Validation.PluralStyle != "i18next-v4" { return fmt.Errorf("unsupported validation.plural_style %q", c.Validation.PluralStyle) } + sourceLocale := c.SourceLocale + if sourceLocale == "" { + sourceLocale = "en" + } + if _, err := localeid.Canonical(sourceLocale); err != nil { + return fmt.Errorf("invalid source locale %q: %w", c.SourceLocale, err) + } if len(c.TargetLocales) == 0 { return fmt.Errorf("target_locales must not be empty") } localeSeen := make(map[string]struct{}, len(c.TargetLocales)) for _, locale := range c.TargetLocales { - if !validLocale.MatchString(locale) { - return fmt.Errorf("invalid target locale %q", locale) + canonical, err := localeid.Canonical(locale) + if err != nil { + return fmt.Errorf("invalid target locale %q: %w", locale, err) } - if _, ok := localeSeen[locale]; ok { + if _, ok := localeSeen[canonical]; ok { return fmt.Errorf("duplicate target locale %q", locale) } - localeSeen[locale] = struct{}{} + localeSeen[canonical] = struct{}{} } + overrideSeen := make(map[string]struct{}, len(c.LLM.LocaleOverrides)) for locale := range c.LLM.LocaleOverrides { - if !validLocale.MatchString(locale) { - return fmt.Errorf("invalid llm locale override %q", locale) + canonical, err := localeid.Canonical(locale) + if err != nil { + return fmt.Errorf("invalid llm locale override %q: %w", locale, err) } - if _, ok := localeSeen[locale]; !ok { + if _, duplicate := overrideSeen[canonical]; duplicate { + return fmt.Errorf("duplicate llm locale override %q", locale) + } + overrideSeen[canonical] = struct{}{} + if _, ok := localeSeen[canonical]; !ok { return fmt.Errorf("llm locale override %q is not in target_locales", locale) } } @@ -276,8 +314,6 @@ func validateAPIKeyEnv(apiKeyEnv string, checked map[string]struct{}) error { return nil } -var validLocale = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) - // EffectiveBundles returns explicit bundles, or a backward-compatible bundle // derived from source_path and the historical sibling target convention. func (c *Config) EffectiveBundles() []Bundle { @@ -298,8 +334,8 @@ func (c *Config) EffectiveBundles() []Bundle { // TargetPath resolves a bundle target for a validated locale. func (b Bundle) TargetPath(locale string) (string, error) { - if !validLocale.MatchString(locale) { - return "", fmt.Errorf("invalid target locale %q", locale) + if _, err := localeid.Canonical(locale); err != nil { + return "", fmt.Errorf("invalid target locale %q: %w", locale, err) } if !strings.Contains(b.Target, "{locale}") { return "", fmt.Errorf("bundle %q target must contain {locale}", b.ID) @@ -307,6 +343,27 @@ func (b Bundle) TargetPath(locale string) (string, error) { return filepath.Clean(strings.ReplaceAll(b.Target, "{locale}", locale)), nil } +func localeOverride(overrides map[string]LLMOverride, requested string) (LLMOverride, bool) { + if override, ok := overrides[requested]; ok { + return override, true + } + canonical, err := localeid.Canonical(requested) + if err != nil { + return LLMOverride{}, false + } + keys := make([]string, 0, len(overrides)) + for key := range overrides { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if candidate, candidateErr := localeid.Canonical(key); candidateErr == nil && candidate == canonical { + return overrides[key], true + } + } + return LLMOverride{}, false +} + // APIKey returns the resolved API key from the environment. func (c *Config) APIKey() string { return os.Getenv(c.LLM.APIKeyEnv) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 039fa66..052e926 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" ) @@ -249,3 +250,82 @@ func TestValidateProjectRejectsLLMOverrideForUnknownLocale(t *testing.T) { t.Fatal("ValidateProject accepted an LLM override for an unconfigured locale") } } + +func TestValidateProjectRejectsInvalidBCP47Locales(t *testing.T) { + tests := []struct { + name string + sourceLocale string + targetLocales []string + want string + }{ + {name: "source", sourceLocale: "en_US", targetLocales: []string{"fr"}, want: "invalid source locale"}, + {name: "target", sourceLocale: "en", targetLocales: []string{"fr_FR"}, want: "invalid target locale"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := &Config{ + SourceLocale: test.sourceLocale, + TargetLocales: test.targetLocales, + SourcePath: filepath.Join("locales", "en.json"), + } + err := cfg.ValidateProject() + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ValidateProject error = %v, want error containing %q", err, test.want) + } + }) + } +} + +func TestValidateProjectRejectsCanonicalDuplicateLocales(t *testing.T) { + cfg := &Config{ + SourceLocale: "en", + TargetLocales: []string{"en-us", "en-US"}, + SourcePath: filepath.Join("locales", "en.json"), + } + err := cfg.ValidateProject() + if err == nil || !strings.Contains(err.Error(), "duplicate target locale") { + t.Fatalf("ValidateProject error = %v, want canonical duplicate error", err) + } +} + +func TestValidateProjectAcceptsBCP47ScriptAndRegion(t *testing.T) { + cfg := &Config{ + SourceLocale: "en-US", + TargetLocales: []string{"sr-Latn-RS"}, + SourcePath: filepath.Join("locales", "en.json"), + } + if err := cfg.ValidateProject(); err != nil { + t.Fatalf("ValidateProject rejected valid BCP 47 locales: %v", err) + } +} + +func TestValidateProjectMatchesCanonicalLLMOverrideLocale(t *testing.T) { + cfg := &Config{ + SourceLocale: "en", + TargetLocales: []string{"pt-br"}, + SourcePath: filepath.Join("locales", "en.json"), + LLM: LLM{LocaleOverrides: map[string]LLMOverride{ + "pt-BR": {Provider: "openrouter", Model: "example/model"}, + }}, + } + if err := cfg.ValidateProject(); err != nil { + t.Fatalf("ValidateProject rejected canonical-equivalent override: %v", err) + } + if got := cfg.LLMForLocale("pt-br"); got.Provider != "openrouter" || got.Model != "example/model" { + t.Fatalf("LLMForLocale = %#v, want canonical-equivalent override", got) + } + if !cfg.HasLLMOverrideForLocale("pt-br") { + t.Fatal("HasLLMOverrideForLocale missed canonical-equivalent override") + } + if got, ok := cfg.ConfiguredTargetLocale("pt-BR"); !ok || got != "pt-br" { + t.Fatalf("ConfiguredTargetLocale = %q, %v; want %q, true", got, ok, "pt-br") + } +} + +func TestBundleTargetPathStillRejectsUnsafeLocale(t *testing.T) { + bundle := Bundle{ID: "app", Target: filepath.Join("locales", "{locale}.json")} + if _, err := bundle.TargetPath("../fr"); err == nil { + t.Fatal("TargetPath accepted path traversal as a locale") + } +} diff --git a/internal/locale/locale.go b/internal/locale/locale.go new file mode 100644 index 0000000..bdfe712 --- /dev/null +++ b/internal/locale/locale.go @@ -0,0 +1,209 @@ +// Package locale owns canonical locale identity and CLDR plural metadata. +package locale + +import ( + "fmt" + "slices" + "strings" + "sync" + + "golang.org/x/text/feature/plural" + "golang.org/x/text/language" +) + +// PluralType selects cardinal or ordinal CLDR plural rules. +type PluralType string + +const ( + Cardinal PluralType = "cardinal" + Ordinal PluralType = "ordinal" +) + +var categoryCache sync.Map + +// Canonical validates a BCP 47 language tag and returns its canonical form. +func Canonical(input string) (string, error) { + tag, err := Parse(input) + if err != nil { + return "", err + } + return tag.String(), nil +} + +// Parse validates a BCP 47 language tag. Underscore aliases are deliberately +// rejected so configuration and persisted state have one unambiguous syntax. +func Parse(input string) (language.Tag, error) { + if input == "" { + return language.Tag{}, fmt.Errorf("locale must not be empty") + } + if strings.ContainsRune(input, '_') { + return language.Tag{}, fmt.Errorf("locale must use BCP 47 hyphen separators") + } + tag, err := language.Parse(input) + if err != nil { + return language.Tag{}, fmt.Errorf("invalid BCP 47 language tag: %w", err) + } + return tag, nil +} + +// CardinalCategories returns the CLDR cardinal categories used by a locale. +func CardinalCategories(input string) ([]string, error) { + return categories(input, Cardinal) +} + +// OrdinalCategories returns the CLDR ordinal categories used by a locale. +func OrdinalCategories(input string) ([]string, error) { + return categories(input, Ordinal) +} + +func categories(input string, pluralType PluralType) ([]string, error) { + tag, err := Parse(input) + if err != nil { + return nil, err + } + key := string(pluralType) + ":" + tag.String() + if cached, ok := categoryCache.Load(key); ok { + return slices.Clone(cached.([]string)), nil + } + + rules := plural.Cardinal + if pluralType == Ordinal { + rules = plural.Ordinal + } + seen := make(map[plural.Form]struct{}, 6) + record := func(form plural.Form) { seen[form] = struct{}{} } + + // x/text's generated CLDR evaluator differentiates integer rules through + // mod 100 plus a few documented large-number exceptions. Cover both the + // complete modulus and those exception ranges. + for integer := 0; integer <= 2000; integer++ { + record(rules.MatchPlural(tag, integer, 0, 0, 0, 0)) + } + for _, integer := range []int{10_000, 100_000, 1_000_000, 2_000_000, 10_000_000} { + record(rules.MatchPlural(tag, integer, 0, 0, 0, 0)) + } + + // Decimal rules can distinguish the number of visible digits and fraction + // operands. Values through 199 cover both sides of x/text's bounded rule + // representation and all modulo-100 classes. + for visibleDigits := 1; visibleDigits <= 4; visibleDigits++ { + maxFraction := 1 + for range visibleDigits { + maxFraction *= 10 + } + maxFraction-- + if maxFraction > 199 { + maxFraction = 199 + } + for integer := 0; integer <= 200; integer++ { + for fraction := 0; fraction <= maxFraction; fraction++ { + record(rules.MatchPlural(tag, integer, visibleDigits, visibleDigits, fraction, fraction)) + } + } + } + + // x/text/feature/plural still embeds CLDR 32 rules. Overlay category-set + // changes needed by the current CLDR 49 contract; the public evaluator also + // cannot represent the compact-exponent operand used by "many" rules. + if pluralType == Cardinal && hasCompactExponentMany(tag) { + seen[plural.Many] = struct{}{} + } + if pluralType == Cardinal && hasCurrentCardinalTwo(tag) { + seen[plural.Two] = struct{}{} + } + if pluralType == Cardinal && needsCurrentCardinalOne(tag) { + seen[plural.One] = struct{}{} + } + if pluralType == Cardinal && needsCurrentCardinalZeroFewMany(tag) { + seen[plural.Zero] = struct{}{} + seen[plural.Few] = struct{}{} + seen[plural.Many] = struct{}{} + } + if pluralType == Ordinal && needsCurrentOrdinalMany(tag) { + seen[plural.Many] = struct{}{} + } + if pluralType == Ordinal && needsCurrentOrdinalOne(tag) { + seen[plural.One] = struct{}{} + } + if pluralType == Cardinal && hasRetiredCardinalMany(tag) { + delete(seen, plural.Many) + } + + ordered := make([]string, 0, len(seen)) + for _, candidate := range []struct { + form plural.Form + name string + }{ + {plural.Zero, "zero"}, + {plural.One, "one"}, + {plural.Two, "two"}, + {plural.Few, "few"}, + {plural.Many, "many"}, + {plural.Other, "other"}, + } { + if _, ok := seen[candidate.form]; ok { + ordered = append(ordered, candidate.name) + } + } + if !slices.Contains(ordered, "other") { + ordered = append(ordered, "other") + } + categoryCache.Store(key, slices.Clone(ordered)) + return ordered, nil +} + +func hasCompactExponentMany(tag language.Tag) bool { + base, _ := tag.Base() + switch base.String() { + case "ca", "es", "fr", "gl", "it", "pt", "scn", "vec": + return true + default: + return false + } +} + +func hasRetiredCardinalMany(tag language.Tag) bool { + base, _ := tag.Base() + // Modern CLDR removed Hebrew's former 20/30/... "many" category. Keep the + // current category set even though x/text still evaluates the older rule. + return base.String() == "he" +} + +func hasCurrentCardinalTwo(tag language.Tag) bool { + base, _ := tag.Base() + // CLDR 42 split Maltese cardinal "other" to add a distinct "two" form. + return base.String() == "mt" +} + +func needsCurrentCardinalOne(tag language.Tag) bool { + base, _ := tag.Base() + // These languages postdate x/text's embedded plural tables. + switch base.String() { + case "scn", "vec": + return true + default: + return false + } +} + +func needsCurrentCardinalZeroFewMany(tag language.Tag) bool { + base, _ := tag.Base() + // Modern Cornish distinguishes all six cardinal categories. + return base.String() == "kw" +} + +func needsCurrentOrdinalMany(tag language.Tag) bool { + base, _ := tag.Base() + // Sicilian and Venetian ordinal rules postdate x/text's embedded tables. + switch base.String() { + case "kw", "scn", "vec": + return true + default: + return false + } +} + +func needsCurrentOrdinalOne(tag language.Tag) bool { + base, _ := tag.Base() + return base.String() == "kw" +} diff --git a/internal/locale/locale_test.go b/internal/locale/locale_test.go new file mode 100644 index 0000000..5fa51ac --- /dev/null +++ b/internal/locale/locale_test.go @@ -0,0 +1,125 @@ +package locale + +import ( + "slices" + "testing" +) + +func TestCanonical(t *testing.T) { + tests := map[string]string{ + "EN-us": "en-US", + "sr-latn-rs": "sr-Latn-RS", + "zh-hant-tw": "zh-Hant-TW", + } + for input, want := range tests { + t.Run(input, func(t *testing.T) { + got, err := Canonical(input) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("Canonical(%q) = %q, want %q", input, got, want) + } + }) + } +} + +func TestCanonicalRejectsMalformedTags(t *testing.T) { + for _, input := range []string{"", "en_US", "../fr", "en--US"} { + t.Run(input, func(t *testing.T) { + if _, err := Canonical(input); err == nil { + t.Fatalf("Canonical(%q) succeeded", input) + } + }) + } +} + +func TestCardinalCategoriesUseCLDRRules(t *testing.T) { + tests := map[string][]string{ + "ar": {"zero", "one", "two", "few", "many", "other"}, + "ja": {"other"}, + "ru": {"one", "few", "many", "other"}, + "te": {"one", "other"}, + } + for tag, want := range tests { + t.Run(tag, func(t *testing.T) { + got, err := CardinalCategories(tag) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got, want) { + t.Fatalf("CardinalCategories(%q) = %v, want %v", tag, got, want) + } + }) + } +} + +func TestOrdinalCategoriesUseCLDRRules(t *testing.T) { + got, err := OrdinalCategories("en-US") + if err != nil { + t.Fatal(err) + } + want := []string{"one", "two", "few", "other"} + if !slices.Equal(got, want) { + t.Fatalf("OrdinalCategories(en-US) = %v, want %v", got, want) + } +} + +func TestSupportedLocalesMatchCLDR49CategorySets(t *testing.T) { + tests := map[string]struct { + cardinal []string + ordinal []string + }{ + "ar": {[]string{"zero", "one", "two", "few", "many", "other"}, []string{"other"}}, + "bn": {[]string{"one", "other"}, []string{"one", "two", "few", "many", "other"}}, + "cs": {[]string{"one", "few", "many", "other"}, []string{"other"}}, + "da": {[]string{"one", "other"}, []string{"other"}}, + "de": {[]string{"one", "other"}, []string{"other"}}, + "el": {[]string{"one", "other"}, []string{"other"}}, + "es": {[]string{"one", "many", "other"}, []string{"other"}}, + "fi": {[]string{"one", "other"}, []string{"other"}}, + "fr": {[]string{"one", "many", "other"}, []string{"one", "other"}}, + "he": {[]string{"one", "two", "other"}, []string{"other"}}, + "hi": {[]string{"one", "other"}, []string{"one", "two", "few", "many", "other"}}, + "id": {[]string{"other"}, []string{"other"}}, + "it": {[]string{"one", "many", "other"}, []string{"many", "other"}}, + "ja": {[]string{"other"}, []string{"other"}}, + "ko": {[]string{"other"}, []string{"other"}}, + "kw": {[]string{"zero", "one", "two", "few", "many", "other"}, []string{"one", "many", "other"}}, + "ms": {[]string{"other"}, []string{"one", "other"}}, + "mt": {[]string{"one", "two", "few", "many", "other"}, []string{"other"}}, + "nl": {[]string{"one", "other"}, []string{"other"}}, + "pa": {[]string{"one", "other"}, []string{"other"}}, + "pl": {[]string{"one", "few", "many", "other"}, []string{"other"}}, + "pt-BR": {[]string{"one", "many", "other"}, []string{"other"}}, + "ro": {[]string{"one", "few", "other"}, []string{"one", "other"}}, + "ru": {[]string{"one", "few", "many", "other"}, []string{"other"}}, + "scn": {[]string{"one", "many", "other"}, []string{"many", "other"}}, + "sv": {[]string{"one", "other"}, []string{"one", "other"}}, + "te": {[]string{"one", "other"}, []string{"other"}}, + "th": {[]string{"other"}, []string{"other"}}, + "tr": {[]string{"one", "other"}, []string{"other"}}, + "uk": {[]string{"one", "few", "many", "other"}, []string{"few", "other"}}, + "vec": {[]string{"one", "many", "other"}, []string{"many", "other"}}, + "vi": {[]string{"other"}, []string{"one", "other"}}, + "yue": {[]string{"other"}, []string{"other"}}, + "zh-CN": {[]string{"other"}, []string{"other"}}, + "zh-TW": {[]string{"other"}, []string{"other"}}, + } + + for tag, want := range tests { + t.Run(tag, func(t *testing.T) { + cardinal, err := CardinalCategories(tag) + if err != nil { + t.Fatal(err) + } + ordinal, err := OrdinalCategories(tag) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(cardinal, want.cardinal) || !slices.Equal(ordinal, want.ordinal) { + t.Fatalf("%s categories: cardinal=%v ordinal=%v, want cardinal=%v ordinal=%v", tag, cardinal, ordinal, want.cardinal, want.ordinal) + } + }) + } +} diff --git a/internal/message/message.go b/internal/message/message.go new file mode 100644 index 0000000..9a5496f --- /dev/null +++ b/internal/message/message.go @@ -0,0 +1,748 @@ +// Package message parses and compares ICU MessageFormat messages. +package message + +import ( + "fmt" + "sort" + "strconv" + "strings" + "unicode" + + localeid "github.com/Tom-R-Main/Internationalizer/internal/locale" +) + +// Code is a stable machine-readable message validation outcome. +type Code string + +const ( + CodeSyntax Code = "syntax" + CodeArgumentMismatch Code = "argument_mismatch" + CodeArgumentTypeMismatch Code = "argument_type_mismatch" + CodeArgumentStyleMismatch Code = "argument_style_mismatch" + CodeSelectorMismatch Code = "selector_mismatch" + CodeInvalidPluralCategory Code = "invalid_plural_category" +) + +const maxNestingDepth = 100 + +// Issue describes one structural incompatibility between source and target. +type Issue struct { + Code Code + Argument string + Message string +} + +// ArgumentType identifies the ICU operation performed by an argument. +type ArgumentType string + +const ( + ArgumentSimple ArgumentType = "" + ArgumentSelect ArgumentType = "select" + ArgumentPlural ArgumentType = "plural" + ArgumentSelectOrdinal ArgumentType = "selectordinal" + ArgumentNumber ArgumentType = "number" + ArgumentDate ArgumentType = "date" + ArgumentTime ArgumentType = "time" +) + +type elementKind uint8 + +const ( + elementText elementKind = iota + elementArgument + elementPound +) + +// Message is a parsed ICU message. Its String method emits a deterministic +// representation suitable for structural hashing. +type Message struct { + elements []element +} + +type element struct { + kind elementKind + text string + argument *Argument +} + +// Argument is one parsed ICU argument. +type Argument struct { + Name string + Type ArgumentType + Style string + Offset int + Options []Option +} + +// Option is one select or plural branch. +type Option struct { + Selector string + Message *Message +} + +// Parse parses an ICU MessageFormat message. +func Parse(input string) (*Message, error) { + parser := parser{input: input} + message, err := parser.parseMessage(false, false) + if err != nil { + return nil, err + } + if parser.position != len(input) { + return nil, parser.errorf("unexpected trailing input") + } + return message, nil +} + +// LooksLike reports whether input contains an ICU-shaped argument. It excludes +// the other interpolation syntaxes Internationalizer already supports. +func LooksLike(input string) bool { + for index := 0; index < len(input); index++ { + if input[index] != '{' || (index > 0 && (input[index-1] == '{' || input[index-1] == '%')) || (index+1 < len(input) && input[index+1] == '{') { + continue + } + cursor := index + 1 + for cursor < len(input) && isSpace(input[cursor]) { + cursor++ + } + start := cursor + for cursor < len(input) && isIdentifierByte(input[cursor]) { + cursor++ + } + if cursor == start { + continue + } + for cursor < len(input) && isSpace(input[cursor]) { + cursor++ + } + if cursor >= len(input) || input[cursor] == '}' || input[cursor] == ',' { + return true + } + } + return false +} + +// Compare reports structural incompatibilities between ICU-shaped source and +// target messages. Plain text and non-ICU interpolation are ignored. +func Compare(source, target, targetLocale string) []Issue { + if !LooksLike(source) && !LooksLike(target) { + return nil + } + sourceMessage, err := Parse(source) + if err != nil { + return []Issue{{Code: CodeSyntax, Message: fmt.Sprintf("source ICU message: %v", err)}} + } + targetMessage, err := Parse(target) + if err != nil { + return []Issue{{Code: CodeSyntax, Message: fmt.Sprintf("target ICU message: %v", err)}} + } + + issues := compareMessages(sourceMessage, targetMessage, targetLocale, "") + sort.Slice(issues, func(left, right int) bool { + if issues[left].Argument != issues[right].Argument { + return issues[left].Argument < issues[right].Argument + } + if issues[left].Code != issues[right].Code { + return issues[left].Code < issues[right].Code + } + return issues[left].Message < issues[right].Message + }) + return issues +} + +func compareMessages(source, target *Message, targetLocale, parentArgument string) []Issue { + sourceArguments := immediateArguments(source) + targetArguments := immediateArguments(target) + var issues []Issue + if sourcePounds, targetPounds := poundCount(source), poundCount(target); sourcePounds != targetPounds { + issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: parentArgument, Message: fmt.Sprintf("ICU plural branch contains %d pound placeholders in the source and %d in the target", sourcePounds, targetPounds)}) + } + for name, sourceOccurrences := range sourceArguments { + targetOccurrences, ok := targetArguments[name] + if !ok { + issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q is missing from the corresponding target branch", name)}) + continue + } + if len(sourceOccurrences) != len(targetOccurrences) { + issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q occurs %d times in the source branch and %d times in the target branch", name, len(sourceOccurrences), len(targetOccurrences))}) + } + if len(sourceOccurrences) == len(targetOccurrences) && hasPerfectArgumentMatching(sourceOccurrences, targetOccurrences, targetLocale) { + continue + } + sourceOccurrences = sortedArguments(sourceOccurrences) + targetOccurrences = sortedArguments(targetOccurrences) + occurrences := min(len(sourceOccurrences), len(targetOccurrences)) + for index := range occurrences { + issues = append(issues, compareArguments(sourceOccurrences[index], targetOccurrences[index], targetLocale)...) + } + } + for name := range targetArguments { + if _, ok := sourceArguments[name]; !ok { + issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q is absent from the corresponding source branch", name)}) + } + } + return issues +} + +func hasPerfectArgumentMatching(source, target []*Argument, targetLocale string) bool { + matchedSource := make([]int, len(target)) + for index := range matchedSource { + matchedSource[index] = -1 + } + var match func(int, []bool) bool + match = func(sourceIndex int, visited []bool) bool { + for targetIndex := range target { + if visited[targetIndex] || len(compareArguments(source[sourceIndex], target[targetIndex], targetLocale)) != 0 { + continue + } + visited[targetIndex] = true + if matchedSource[targetIndex] == -1 || match(matchedSource[targetIndex], visited) { + matchedSource[targetIndex] = sourceIndex + return true + } + } + return false + } + for sourceIndex := range source { + if !match(sourceIndex, make([]bool, len(target))) { + return false + } + } + return true +} + +func sortedArguments(arguments []*Argument) []*Argument { + result := append([]*Argument(nil), arguments...) + sort.SliceStable(result, func(left, right int) bool { + return argumentSortKey(result[left]) < argumentSortKey(result[right]) + }) + return result +} + +func argumentSortKey(argument *Argument) string { + options := make([]string, 0, len(argument.Options)) + for _, option := range argument.Options { + options = append(options, option.Selector+"{"+messageSortKey(option.Message)+"}") + } + sort.Strings(options) + return fmt.Sprintf("%s\x00%s\x00%020d\x00%s", argument.Type, argument.Style, argument.Offset, strings.Join(options, "\x00")) +} + +func messageSortKey(message *Message) string { + parts := make([]string, 0, len(message.elements)) + for _, element := range message.elements { + switch element.kind { + case elementArgument: + parts = append(parts, element.argument.Name+"{"+argumentSortKey(element.argument)+"}") + case elementPound: + parts = append(parts, "#") + } + } + sort.Strings(parts) + return strings.Join(parts, "\x00") +} + +func poundCount(message *Message) int { + count := 0 + for _, element := range message.elements { + if element.kind == elementPound { + count++ + } + } + return count +} + +func compareArguments(source, target *Argument, targetLocale string) []Issue { + name := source.Name + if source.Type != target.Type { + return []Issue{{Code: CodeArgumentTypeMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q types differ: source %q, target %q", name, displayType(source.Type), displayType(target.Type))}} + } + var issues []Issue + switch source.Type { + case ArgumentNumber, ArgumentDate, ArgumentTime: + if source.Style != target.Style { + issues = append(issues, Issue{Code: CodeArgumentStyleMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q formatter styles differ", name)}) + } + case ArgumentSelect: + sourceOptions := optionsBySelector(source) + targetOptions := optionsBySelector(target) + if !sameOptionSet(sourceOptions, targetOptions) { + issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: name, Message: fmt.Sprintf("ICU select argument %q selectors differ", name)}) + } + for selector, sourceBranch := range sourceOptions { + if targetBranch, ok := targetOptions[selector]; ok { + issues = append(issues, compareMessages(sourceBranch, targetBranch, targetLocale, name)...) + } + } + case ArgumentPlural, ArgumentSelectOrdinal: + if source.Offset != target.Offset { + issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q plural offsets differ", name)}) + } + issues = append(issues, comparePluralSelectors(source, target, targetLocale)...) + sourceOptions := optionsBySelector(source) + targetOptions := optionsBySelector(target) + for selector, sourceBranch := range sourceOptions { + if targetBranch, ok := targetOptions[selector]; ok { + issues = append(issues, compareMessages(sourceBranch, targetBranch, targetLocale, name)...) + } + } + sourceOther := sourceOptions["other"] + for selector, targetBranch := range targetOptions { + if _, sourceHasSelector := sourceOptions[selector]; !sourceHasSelector && !isExactSelector(selector) { + issues = append(issues, compareMessages(sourceOther, targetBranch, targetLocale, name)...) + } + } + } + return issues +} + +func comparePluralSelectors(source, target *Argument, targetLocale string) []Issue { + name := source.Name + categories, err := localeid.CardinalCategories(targetLocale) + if source.Type == ArgumentSelectOrdinal { + categories, err = localeid.OrdinalCategories(targetLocale) + } + if err != nil { + return []Issue{{Code: CodeInvalidPluralCategory, Argument: name, Message: fmt.Sprintf("cannot resolve plural categories for locale %q: %v", targetLocale, err)}} + } + allowed := make(map[string]struct{}, len(categories)) + for _, category := range categories { + allowed[category] = struct{}{} + } + sourceSelectors := selectorSet(source) + targetSelectors := selectorSet(target) + var issues []Issue + for selector := range targetSelectors { + if isExactSelector(selector) { + continue + } + if _, ok := allowed[selector]; !ok { + issues = append(issues, Issue{Code: CodeInvalidPluralCategory, Argument: name, Message: fmt.Sprintf("ICU argument %q plural category %q is not used by locale %q", name, selector, targetLocale)}) + } + } + for selector := range sourceSelectors { + _, targetHasSelector := targetSelectors[selector] + if targetHasSelector { + continue + } + _, applicableCategory := allowed[selector] + if isExactSelector(selector) || applicableCategory { + issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q target is missing selector %q", name, selector)}) + } + } + return issues +} + +func immediateArguments(message *Message) map[string][]*Argument { + result := make(map[string][]*Argument) + for _, element := range message.elements { + if element.kind == elementArgument { + argument := element.argument + result[argument.Name] = append(result[argument.Name], argument) + } + } + return result +} + +func optionsBySelector(argument *Argument) map[string]*Message { + result := make(map[string]*Message, len(argument.Options)) + for _, option := range argument.Options { + result[option.Selector] = option.Message + } + return result +} + +func selectorSet(argument *Argument) map[string]struct{} { + result := make(map[string]struct{}, len(argument.Options)) + for _, option := range argument.Options { + result[option.Selector] = struct{}{} + } + return result +} + +func sameOptionSet(left, right map[string]*Message) bool { + if len(left) != len(right) { + return false + } + for value := range left { + if _, ok := right[value]; !ok { + return false + } + } + return true +} + +func displayType(argumentType ArgumentType) string { + if argumentType == ArgumentSimple { + return "argument" + } + return string(argumentType) +} + +func isExactSelector(selector string) bool { + if !strings.HasPrefix(selector, "=") || len(selector) == 1 { + return false + } + _, err := strconv.ParseFloat(selector[1:], 64) + return err == nil +} + +type parser struct { + input string + position int + depth int +} + +func (p *parser) parseBranch(pluralContext bool) (*Message, error) { + if p.depth >= maxNestingDepth { + return nil, p.errorf("ICU message nesting exceeds %d levels", maxNestingDepth) + } + p.depth++ + defer func() { p.depth-- }() + return p.parseMessage(true, pluralContext) +} + +func (p *parser) parseMessage(nested, pluralContext bool) (*Message, error) { + message := &Message{} + var literal strings.Builder + flush := func() { + if literal.Len() == 0 { + return + } + message.elements = append(message.elements, element{kind: elementText, text: literal.String()}) + literal.Reset() + } + + for p.position < len(p.input) { + switch current := p.input[p.position]; current { + case '{': + flush() + argument, err := p.parseArgument() + if err != nil { + return nil, err + } + message.elements = append(message.elements, element{kind: elementArgument, argument: argument}) + case '}': + if !nested { + return nil, p.errorf("unexpected closing brace") + } + flush() + return message, nil + case '#': + if pluralContext { + flush() + message.elements = append(message.elements, element{kind: elementPound}) + p.position++ + continue + } + literal.WriteByte(current) + p.position++ + case '\'': + quoted, err := p.parseApostrophe(pluralContext) + if err != nil { + return nil, err + } + literal.WriteString(quoted) + default: + literal.WriteByte(current) + p.position++ + } + } + if nested { + return nil, p.errorf("unterminated message branch") + } + flush() + return message, nil +} + +func (p *parser) parseApostrophe(pluralContext bool) (string, error) { + p.position++ + if p.position < len(p.input) && p.input[p.position] == '\'' { + p.position++ + return "'", nil + } + if p.position >= len(p.input) || !p.apostropheStartsQuote(pluralContext) { + return "'", nil + } + var quoted strings.Builder + for p.position < len(p.input) { + if p.input[p.position] != '\'' { + quoted.WriteByte(p.input[p.position]) + p.position++ + continue + } + if p.position+1 < len(p.input) && p.input[p.position+1] == '\'' { + quoted.WriteByte('\'') + p.position += 2 + continue + } + p.position++ + return quoted.String(), nil + } + return "", p.errorf("unterminated apostrophe quote") +} + +func (p *parser) apostropheStartsQuote(pluralContext bool) bool { + switch p.input[p.position] { + case '{', '}': + return true + case '#': + return pluralContext + default: + return false + } +} + +func (p *parser) parseArgument() (*Argument, error) { + p.position++ + p.skipSpace() + name := p.readIdentifier() + if name == "" { + return nil, p.errorf("argument name is required") + } + p.skipSpace() + if p.consume('}') { + return &Argument{Name: name}, nil + } + if !p.consume(',') { + return nil, p.errorf("expected comma or closing brace after argument %q", name) + } + p.skipSpace() + argumentType := ArgumentType(strings.ToLower(p.readIdentifier())) + switch argumentType { + case ArgumentSelect, ArgumentPlural, ArgumentSelectOrdinal, ArgumentNumber, ArgumentDate, ArgumentTime: + default: + return nil, p.errorf("unsupported argument type %q", argumentType) + } + argument := &Argument{Name: name, Type: argumentType} + p.skipSpace() + if argumentType == ArgumentNumber || argumentType == ArgumentDate || argumentType == ArgumentTime { + if p.consume(',') { + styleStart := p.position + for p.position < len(p.input) && p.input[p.position] != '}' { + if p.input[p.position] == '{' { + return nil, p.errorf("unexpected opening brace in %s style", argumentType) + } + p.position++ + } + argument.Style = strings.TrimSpace(p.input[styleStart:p.position]) + if argument.Style == "" { + return nil, p.errorf("%s style must not be empty", argumentType) + } + } + if !p.consume('}') { + return nil, p.errorf("unterminated %s argument %q", argumentType, name) + } + return argument, nil + } + + if !p.consume(',') { + return nil, p.errorf("expected comma before selectors for argument %q", name) + } + p.skipSpace() + if argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal { + if strings.HasPrefix(p.input[p.position:], "offset:") { + p.position += len("offset:") + start := p.position + for p.position < len(p.input) && p.input[p.position] >= '0' && p.input[p.position] <= '9' { + p.position++ + } + if start == p.position { + return nil, p.errorf("plural offset must be a non-negative integer") + } + offset, err := strconv.Atoi(p.input[start:p.position]) + if err != nil { + return nil, p.errorf("invalid plural offset") + } + argument.Offset = offset + p.skipSpace() + } + } + + selectors := make(map[string]struct{}) + for p.position < len(p.input) && p.input[p.position] != '}' { + selector := p.readSelector() + if selector == "" { + return nil, p.errorf("selector is required for argument %q", name) + } + if _, duplicate := selectors[selector]; duplicate { + return nil, p.errorf("duplicate selector %q for argument %q", selector, name) + } + if argumentType != ArgumentSelect && !isPluralSelector(selector) { + return nil, p.errorf("invalid plural selector %q", selector) + } + selectors[selector] = struct{}{} + p.skipSpace() + if !p.consume('{') { + return nil, p.errorf("expected message branch for selector %q", selector) + } + branch, err := p.parseBranch(argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal) + if err != nil { + return nil, err + } + if !p.consume('}') { + return nil, p.errorf("unterminated selector %q", selector) + } + argument.Options = append(argument.Options, Option{Selector: selector, Message: branch}) + p.skipSpace() + } + if !p.consume('}') { + return nil, p.errorf("unterminated argument %q", name) + } + if _, ok := selectors["other"]; !ok { + return nil, p.errorf("argument %q requires an \"other\" selector", name) + } + return argument, nil +} + +func (p *parser) readIdentifier() string { + start := p.position + for p.position < len(p.input) && isIdentifierByte(p.input[p.position]) { + p.position++ + } + return p.input[start:p.position] +} + +func (p *parser) readSelector() string { + p.skipSpace() + start := p.position + for p.position < len(p.input) && !isSpace(p.input[p.position]) && p.input[p.position] != '{' && p.input[p.position] != '}' { + p.position++ + } + return p.input[start:p.position] +} + +func (p *parser) skipSpace() { + for p.position < len(p.input) && isSpace(p.input[p.position]) { + p.position++ + } +} + +func (p *parser) consume(expected byte) bool { + if p.position >= len(p.input) || p.input[p.position] != expected { + return false + } + p.position++ + return true +} + +func (p *parser) errorf(format string, arguments ...any) error { + return fmt.Errorf("at byte %d: %s", p.position, fmt.Sprintf(format, arguments...)) +} + +func isIdentifierByte(value byte) bool { + return value == '_' || value == '-' || value == '.' || value >= '0' && value <= '9' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' +} + +func isSpace(value byte) bool { + return unicode.IsSpace(rune(value)) +} + +func isPluralSelector(selector string) bool { + if isExactSelector(selector) { + return true + } + switch selector { + case "zero", "one", "two", "few", "many", "other": + return true + default: + return false + } +} + +// String emits a deterministic ICU representation. +func (message *Message) String() string { + var output strings.Builder + message.writeTo(&output, false) + return output.String() +} + +func (message *Message) writeTo(output *strings.Builder, pluralContext bool) { + for _, element := range message.elements { + switch element.kind { + case elementText: + writeLiteral(output, element.text, pluralContext) + case elementPound: + output.WriteByte('#') + case elementArgument: + element.argument.writeTo(output) + } + } +} + +func (argument *Argument) writeTo(output *strings.Builder) { + output.WriteByte('{') + output.WriteString(argument.Name) + if argument.Type == ArgumentSimple { + output.WriteByte('}') + return + } + output.WriteString(", ") + output.WriteString(string(argument.Type)) + if argument.Type == ArgumentNumber || argument.Type == ArgumentDate || argument.Type == ArgumentTime { + if argument.Style != "" { + output.WriteString(", ") + output.WriteString(argument.Style) + } + output.WriteByte('}') + return + } + output.WriteString(", ") + if argument.Offset != 0 { + _, _ = fmt.Fprintf(output, "offset:%d ", argument.Offset) + } + options := append([]Option(nil), argument.Options...) + sort.SliceStable(options, func(left, right int) bool { return options[left].Selector < options[right].Selector }) + for index, option := range options { + if index > 0 { + output.WriteByte(' ') + } + output.WriteString(option.Selector) + output.WriteString(" {") + option.Message.writeTo(output, argument.Type == ArgumentPlural || argument.Type == ArgumentSelectOrdinal) + output.WriteByte('}') + } + output.WriteByte('}') +} + +func writeLiteral(output *strings.Builder, literal string, pluralContext bool) { + characters := []rune(literal) + firstQuoted := -1 + lastQuoted := -1 + for index, character := range characters { + if literalNeedsQuote(character, pluralContext) { + if firstQuoted < 0 { + firstQuoted = index + } + lastQuoted = index + } + } + if firstQuoted < 0 { + writeUnquotedLiteral(output, characters) + return + } + writeUnquotedLiteral(output, characters[:firstQuoted]) + output.WriteByte('\'') + for _, character := range characters[firstQuoted : lastQuoted+1] { + if character == '\'' { + output.WriteString("''") + } else { + output.WriteRune(character) + } + } + output.WriteByte('\'') + writeUnquotedLiteral(output, characters[lastQuoted+1:]) +} + +func literalNeedsQuote(character rune, pluralContext bool) bool { + return character == '{' || character == '}' || pluralContext && character == '#' +} + +func writeUnquotedLiteral(output *strings.Builder, characters []rune) { + for _, character := range characters { + if character == '\'' { + output.WriteString("''") + } else { + output.WriteRune(character) + } + } +} diff --git a/internal/message/message_test.go b/internal/message/message_test.go new file mode 100644 index 0000000..1ffa332 --- /dev/null +++ b/internal/message/message_test.go @@ -0,0 +1,222 @@ +package message + +import ( + "slices" + "strings" + "testing" +) + +func TestParseAndPrintNestedICUMessage(t *testing.T) { + input := "{gender, select, female {{count, plural, offset:1 =0 {None} one {She and one other} other {She and # others}}} other {{count, number, ::compact-short} items}}" + parsed, err := Parse(input) + if err != nil { + t.Fatal(err) + } + printed := parsed.String() + reparsed, err := Parse(printed) + if err != nil { + t.Fatalf("printed message is invalid: %v\n%s", err, printed) + } + if reparsed.String() != printed { + t.Fatalf("printer is unstable:\nfirst: %s\nsecond: %s", printed, reparsed.String()) + } +} + +func TestParseRejectsMalformedMessages(t *testing.T) { + tests := []string{ + "{count, plural, one {One}", + "{count, plural, one {One}}", + "{count, plural other {Items}}", + "{kind, select, other {One} other {Two}}", + "{count, plural, offset:nope other {Items}}", + } + for _, input := range tests { + t.Run(input, func(t *testing.T) { + if _, err := Parse(input); err == nil { + t.Fatalf("Parse(%q) succeeded", input) + } + }) + } +} + +func TestParseRejectsExcessiveNesting(t *testing.T) { + input := strings.Repeat("{value, select, other {", maxNestingDepth+1) + "value" + strings.Repeat("}}", maxNestingDepth+1) + if _, err := Parse(input); err == nil || !strings.Contains(err.Error(), "nesting exceeds") { + t.Fatalf("Parse error = %v, want nesting limit error", err) + } +} + +func TestCompareChecksArgumentsAndSelectors(t *testing.T) { + issues := Compare( + "{gender, select, female {{count, plural, one {One} other {#}}} other {They}}", + "{gender, select, female {{count, select, one {Un} other {Plusieurs}}} male {Il} other {Iel}}", + "fr", + ) + got := make([]Code, len(issues)) + for index, issue := range issues { + got[index] = issue.Code + } + if !slices.Contains(got, CodeSelectorMismatch) || !slices.Contains(got, CodeArgumentTypeMismatch) { + t.Fatalf("Compare issues = %#v", issues) + } +} + +func TestCompareRejectsLocaleInapplicablePluralCategory(t *testing.T) { + issues := Compare( + "{count, plural, one {One} other {# items}}", + "{count, plural, one {ఒక అంశం} few {కొన్ని అంశాలు} other {# అంశాలు}}", + "te", + ) + if !hasCode(issues, CodeInvalidPluralCategory) { + t.Fatalf("Compare issues = %#v, want %q", issues, CodeInvalidPluralCategory) + } +} + +func TestCompareAllowsLocaleCategoryAndExactSelector(t *testing.T) { + issues := Compare( + "{count, plural, one {One} other {# items}}", + "{count, plural, =0 {Нет элементов} one {Один элемент} few {# элемента} other {# элементов}}", + "ru", + ) + if len(issues) != 0 { + t.Fatalf("Compare returned issues for valid Russian message: %#v", issues) + } +} + +func TestCompareChecksTargetOnlyLocaleCategoryAgainstSourceOther(t *testing.T) { + source := "{count, plural, one {{name} has one item} other {{name} has # items}}" + valid := "{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} other {{name} имеет # предметов}}" + if issues := Compare(source, valid, "ru"); len(issues) != 0 { + t.Fatalf("Compare rejected structurally valid target-only category: %#v", issues) + } + + invalid := "{count, plural, one {{name} имеет один предмет} few {несколько предметов} other {{name} имеет # предметов}}" + issues := Compare(source, invalid, "ru") + if !hasCode(issues, CodeArgumentMismatch) || !hasCode(issues, CodeSelectorMismatch) { + t.Fatalf("Compare issues = %#v, want missing argument and pound placeholder", issues) + } +} + +func TestComparePairsRepeatedPluralsWithTargetOnlyCategories(t *testing.T) { + source := "{n, plural, one {{a}} other {{a} #}} then {n, plural, one {{z}} other {{z} #}}" + target := "{n, plural, one {{z}} few {{z} #} other {{z} #}} puis {n, plural, one {{a}} other {{a} #}}" + if issues := Compare(source, target, "ru"); len(issues) != 0 { + t.Fatalf("Compare rejected valid repeated plural reordering: %#v", issues) + } +} + +func TestComparePreservesPluralOffsetsAndFormatterStyles(t *testing.T) { + tests := []struct { + name string + source string + target string + code Code + }{ + { + name: "plural offset", + source: "{count, plural, offset:1 one {One} other {#}}", + target: "{count, plural, one {Un} other {#}}", + code: CodeSelectorMismatch, + }, + { + name: "number style", + source: "Total: {amount, number, ::currency/USD}", + target: "Total : {amount, number, ::percent}", + code: CodeArgumentStyleMismatch, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + issues := Compare(test.source, test.target, "fr") + if !hasCode(issues, test.code) { + t.Fatalf("Compare issues = %#v, want %q", issues, test.code) + } + }) + } +} + +func TestComparePreservesArgumentsWithinCorrespondingBranches(t *testing.T) { + source := "{gender, select, female {{count, plural, offset:1 one {One} other {#}}} other {{count, plural, one {One} other {#}}}}" + target := "{gender, select, female {{count, plural, one {Une} other {#}}} other {{count, plural, offset:1 one {Une} other {#}}}}" + issues := Compare(source, target, "fr") + if !hasCode(issues, CodeSelectorMismatch) { + t.Fatalf("Compare issues = %#v, want branch-specific offset mismatch", issues) + } +} + +func TestCompareAllowsRepeatedArgumentReorderingWithinBranch(t *testing.T) { + source := "Created {date, date, short}; archived {date, date, long}" + target := "Archivé {date, date, long}; créé {date, date, short}" + if issues := Compare(source, target, "fr"); len(issues) != 0 { + t.Fatalf("Compare rejected valid repeated-argument reordering: %#v", issues) + } +} + +func TestCompareAllowsRepeatedNestedArgumentReorderingWithinBranch(t *testing.T) { + source := "{choice, select, first {{first}} other {A}} then {choice, select, first {{last}} other {B}}" + target := "{choice, select, first {{last}} other {B traduit}} puis {choice, select, first {{first}} other {A traduit}}" + if issues := Compare(source, target, "fr"); len(issues) != 0 { + t.Fatalf("Compare rejected valid repeated nested-argument reordering: %#v", issues) + } +} + +func TestComparePreservesPoundPlaceholders(t *testing.T) { + issues := Compare( + "{count, plural, one {One item} other {# items}}", + "{count, plural, one {Un article} other {articles}}", + "fr", + ) + if !hasCode(issues, CodeSelectorMismatch) { + t.Fatalf("Compare issues = %#v, want missing pound placeholder", issues) + } +} + +func TestCompareReportsMalformedSourceAndTarget(t *testing.T) { + for name, values := range map[string][2]string{ + "source": {"{count, plural, one {One}}", "{count, plural, one {Un} other {Autres}}"}, + "target": {"{count, plural, one {One} other {Other}}", "{count, plural, one {Un} other {Autres}"}, + } { + t.Run(name, func(t *testing.T) { + issues := Compare(values[0], values[1], "fr") + if !hasCode(issues, CodeSyntax) { + t.Fatalf("Compare issues = %#v, want syntax issue", issues) + } + }) + } +} + +func hasCode(issues []Issue, code Code) bool { + for _, issue := range issues { + if issue.Code == code { + return true + } + } + return false +} + +func FuzzParsePrint(f *testing.F) { + for _, seed := range []string{ + "plain text", + "Hello {name}", + "{gender, select, female {She} other {They}}", + "{count, plural, offset:1 =0 {None} one {One} other {# items}}", + "{place, selectordinal, one {#st} two {#nd} few {#rd} other {#th}}", + "This '{' is literal and this apostrophe isn''t ambiguous", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, input string) { + parsed, err := Parse(input) + if err != nil { + return + } + printed := parsed.String() + reparsed, err := Parse(printed) + if err != nil { + t.Fatalf("Parse(String(Parse(%q))) failed: %v; printed %q", input, err, printed) + } + if reparsed.String() != printed { + t.Fatalf("unstable print for %q: first %q, second %q", input, printed, reparsed.String()) + } + }) +} diff --git a/internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26 b/internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26 new file mode 100644 index 0000000..c880784 --- /dev/null +++ b/internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("0'{''{'0") diff --git a/internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738 b/internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738 new file mode 100644 index 0000000..25d012b --- /dev/null +++ b/internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("'{{'") diff --git a/internal/policy/policy.go b/internal/policy/policy.go index 3606505..84e72ae 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -5,6 +5,7 @@ import ( "github.com/Tom-R-Main/Internationalizer/internal/config" "github.com/Tom-R-Main/Internationalizer/internal/glossary" "github.com/Tom-R-Main/Internationalizer/internal/llm" + "github.com/Tom-R-Main/Internationalizer/internal/locale" "github.com/Tom-R-Main/Internationalizer/internal/state" ) @@ -22,10 +23,23 @@ type Resolved struct { // already-loaded style-guide and glossary content so policy resolution remains // independent of filesystem layout. func Resolve(cfg *config.Config, targetLocale, format, styleGuide string, terms []glossary.Term) (Resolved, error) { - effectiveLLM := cfg.LLMForLocale(targetLocale) - prompt := llm.BuildSystemPrompt(cfg.SourceLocale, targetLocale, styleGuide, terms) + sourceLocale := cfg.SourceLocale + if sourceLocale == "" { + sourceLocale = "en" + } + canonicalSource, err := locale.Canonical(sourceLocale) + if err != nil { + return Resolved{}, err + } + canonicalTarget, err := locale.Canonical(targetLocale) + if err != nil { + return Resolved{}, err + } + + effectiveLLM := cfg.LLMForLocale(canonicalTarget) + prompt := llm.BuildSystemPrompt(canonicalSource, canonicalTarget, styleGuide, terms) if format == "markdown" { - prompt = llm.BuildDocumentPrompt(cfg.SourceLocale, targetLocale, styleGuide, terms) + prompt = llm.BuildDocumentPrompt(canonicalSource, canonicalTarget, styleGuide, terms) } hash, err := state.HashValue(struct { @@ -37,7 +51,7 @@ func Resolve(cfg *config.Config, targetLocale, format, styleGuide string, terms Model string `json:"model"` Reasoning string `json:"reasoning_effort"` Prompt string `json:"prompt"` - }{promptPolicyVersion, cfg.SourceLocale, targetLocale, format, effectiveLLM.Provider, effectiveLLM.Model, llm.EffectiveReasoningEffort(effectiveLLM), prompt}) + }{promptPolicyVersion, canonicalSource, canonicalTarget, format, effectiveLLM.Provider, effectiveLLM.Model, llm.EffectiveReasoningEffort(effectiveLLM), prompt}) if err != nil { return Resolved{}, err } diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index f020c09..e602acc 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -140,6 +140,25 @@ func TestResolveUsesLocaleLLMOverride(t *testing.T) { } } +func TestResolveCanonicalizesLocaleIdentity(t *testing.T) { + canonicalConfig := baseConfig() + canonicalConfig.SourceLocale = "en-US" + canonical, err := policy.Resolve(canonicalConfig, "pt-BR", "json", "", nil) + if err != nil { + t.Fatal(err) + } + + variantConfig := baseConfig() + variantConfig.SourceLocale = "en-us" + variant, err := policy.Resolve(variantConfig, "pt-br", "json", "", nil) + if err != nil { + t.Fatal(err) + } + if variant.Hash != canonical.Hash || variant.Prompt != canonical.Prompt { + t.Fatal("canonical-equivalent locales produced different translation policies") + } +} + func baseConfig() *config.Config { return &config.Config{ SourceLocale: "en", diff --git a/internal/state/manifest.go b/internal/state/manifest.go index 9910543..a931405 100644 --- a/internal/state/manifest.go +++ b/internal/state/manifest.go @@ -9,6 +9,8 @@ import ( "os" "path/filepath" "time" + + "github.com/Tom-R-Main/Internationalizer/internal/locale" ) const SchemaVersion = 1 @@ -62,6 +64,16 @@ func Load(path string) (*Manifest, error) { if manifest.Translations == nil { manifest.Translations = make(map[string]Entry) } + canonicalTranslations := make(map[string]Entry, len(manifest.Translations)) + for _, entry := range manifest.Translations { + entry.Locale = canonicalLocaleOrOriginal(entry.Locale) + identity := Identity(entry.Bundle, entry.Key, entry.Locale) + if existing, duplicate := canonicalTranslations[identity]; duplicate && existing != entry { + return nil, fmt.Errorf("manifest %s contains conflicting entries for %s/%s/%s", path, entry.Bundle, entry.Key, entry.Locale) + } + canonicalTranslations[identity] = entry + } + manifest.Translations = canonicalTranslations return &manifest, nil } @@ -85,12 +97,21 @@ func (m *Manifest) Set(entry Entry) { if m.Translations == nil { m.Translations = make(map[string]Entry) } + entry.Locale = canonicalLocaleOrOriginal(entry.Locale) m.Translations[Identity(entry.Bundle, entry.Key, entry.Locale)] = entry } // Identity returns a stable full SHA-256 identifier for a logical translation. func Identity(bundle, key, locale string) string { - return hashBytes([]byte(bundle + "\x00" + key + "\x00" + locale)) + return hashBytes([]byte(bundle + "\x00" + key + "\x00" + canonicalLocaleOrOriginal(locale))) +} + +func canonicalLocaleOrOriginal(value string) string { + canonical, err := locale.Canonical(value) + if err != nil { + return value + } + return canonical } // SourceHash binds translation reuse to both content and source format. diff --git a/internal/state/manifest_test.go b/internal/state/manifest_test.go index 64f7146..eafc7b9 100644 --- a/internal/state/manifest_test.go +++ b/internal/state/manifest_test.go @@ -50,6 +50,33 @@ func TestLoadRejectsMalformedManifest(t *testing.T) { } } +func TestLoadMigratesLegacyLocaleIdentityInMemory(t *testing.T) { + path := filepath.Join(t.TempDir(), "manifest.json") + data := []byte(`{"schema_version":1,"translations":{"legacy-raw-key":{"bundle":"app","key":"save","locale":"pt-br","source_hash":"source","policy_hash":"policy","target_hash":"target","updated_at":"2026-01-01T00:00:00Z"}}}`) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + manifest, err := Load(path) + if err != nil { + t.Fatal(err) + } + entry, ok := manifest.Get("app", "save", "pt-BR") + if !ok || entry.Locale != "pt-BR" { + t.Fatalf("legacy entry = %#v, %v; want canonicalized lookup", entry, ok) + } +} + +func TestLoadRejectsConflictingCanonicalLocaleIdentities(t *testing.T) { + path := filepath.Join(t.TempDir(), "manifest.json") + data := []byte(`{"schema_version":1,"translations":{"first":{"bundle":"app","key":"save","locale":"pt-br","source_hash":"one","updated_at":"2026-01-01T00:00:00Z"},"second":{"bundle":"app","key":"save","locale":"pt-BR","source_hash":"two","updated_at":"2026-01-01T00:00:00Z"}}}`) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Fatal("Load accepted conflicting canonical-equivalent identities") + } +} + func TestHashesSeparateSourceFormatAndPolicy(t *testing.T) { if SourceHash("json", "Save") == SourceHash("markdown", "Save") { t.Fatal("source hash ignored format") @@ -59,6 +86,23 @@ func TestHashesSeparateSourceFormatAndPolicy(t *testing.T) { } } +func TestManifestIdentityCanonicalizesLocale(t *testing.T) { + manifest := New() + entry := Entry{Bundle: "app", Key: "save", Locale: "pt-br"} + manifest.Set(entry) + + got, ok := manifest.Get("app", "save", "pt-BR") + if !ok { + t.Fatal("canonical-equivalent locale did not find the manifest entry") + } + if got.Locale != "pt-BR" { + t.Fatalf("stored locale = %q, want %q", got.Locale, "pt-BR") + } + if Identity("app", "save", "pt-br") != Identity("app", "save", "pt-BR") { + t.Fatal("canonical-equivalent locales produced different identities") + } +} + func TestHashValueRejectsUnsupportedValue(t *testing.T) { if _, err := HashValue(func() {}); err == nil { t.Fatal("HashValue accepted an unsupported value") diff --git a/internal/tm/tm.go b/internal/tm/tm.go index d698aa9..48eb151 100644 --- a/internal/tm/tm.go +++ b/internal/tm/tm.go @@ -10,6 +10,8 @@ import ( "path/filepath" "sync" "time" + + "github.com/Tom-R-Main/Internationalizer/internal/locale" ) // Record is a single translation memory entry. @@ -69,6 +71,7 @@ func Load(path string) (*TM, error) { if err := json.Unmarshal(line, &rec); err != nil { return nil, fmt.Errorf("parsing TM %s line %d: %w", path, lineNumber, err) } + rec = canonicalizeRecord(rec) if t.index[rec.Locale] == nil { t.index[rec.Locale] = make(map[string]Record) } @@ -83,7 +86,7 @@ func (t *TM) Lookup(locale, bundle, key, sourceHash, policyHash string) (Record, t.mu.RLock() defer t.mu.RUnlock() - localeMap, ok := t.index[locale] + localeMap, ok := t.index[canonicalLocaleOrOriginal(locale)] if !ok { return Record{}, false } @@ -98,6 +101,7 @@ func (t *TM) Lookup(locale, bundle, key, sourceHash, policyHash string) (Record, func (t *TM) Add(rec Record) error { t.mu.Lock() defer t.mu.Unlock() + rec = canonicalizeRecord(rec) if err := os.MkdirAll(filepath.Dir(t.path), 0o755); err != nil { return fmt.Errorf("creating TM directory: %w", err) @@ -138,6 +142,9 @@ func (t *TM) AddBatch(records []Record) error { t.mu.Lock() defer t.mu.Unlock() + for i := range records { + records[i] = canonicalizeRecord(records[i]) + } if err := os.MkdirAll(filepath.Dir(t.path), 0o755); err != nil { return fmt.Errorf("creating TM directory: %w", err) @@ -273,3 +280,16 @@ func HashSource(source string) string { func recordKey(bundle, key string) string { return bundle + "\x00" + key } + +func canonicalizeRecord(rec Record) Record { + rec.Locale = canonicalLocaleOrOriginal(rec.Locale) + return rec +} + +func canonicalLocaleOrOriginal(value string) string { + canonical, err := locale.Canonical(value) + if err != nil { + return value + } + return canonical +} diff --git a/internal/tm/tm_test.go b/internal/tm/tm_test.go index b3fe19a..e53c9c6 100644 --- a/internal/tm/tm_test.go +++ b/internal/tm/tm_test.go @@ -83,6 +83,33 @@ func TestTMLookupRequiresMatchingPolicy(t *testing.T) { } } +func TestTMLocaleIdentityIsCanonical(t *testing.T) { + path := filepath.Join(t.TempDir(), "tm.jsonl") + memory, err := Load(path) + if err != nil { + t.Fatal(err) + } + record := Record{Bundle: "app", Key: "save", Source: "Save", Target: "Salvar", Locale: "pt-br", Hash: HashSource("Save"), PolicyHash: "policy-v1", Timestamp: time.Now()} + if err := memory.Add(record); err != nil { + t.Fatal(err) + } + got, ok := memory.Lookup("pt-BR", "app", "save", HashSource("Save"), "policy-v1") + if !ok { + t.Fatal("canonical-equivalent locale did not reuse the translation") + } + if got.Locale != "pt-BR" { + t.Fatalf("stored locale = %q, want %q", got.Locale, "pt-BR") + } + + reloaded, err := Load(path) + if err != nil { + t.Fatal(err) + } + if _, ok := reloaded.Lookup("pt-BR", "app", "save", HashSource("Save"), "policy-v1"); !ok { + t.Fatal("canonical-equivalent locale did not reuse the translation after reload") + } +} + func TestLoadRejectsMalformedRecord(t *testing.T) { path := filepath.Join(t.TempDir(), "tm.jsonl") if err := os.WriteFile(path, []byte("{\n"), 0o644); err != nil { diff --git a/internal/translate/protected.go b/internal/translate/protected.go index 345f58e..dc25d6a 100644 --- a/internal/translate/protected.go +++ b/internal/translate/protected.go @@ -10,12 +10,15 @@ import ( // validateTranslationValue enforces deterministic invariants at both provider // and explicit-adoption boundaries. Instructions in a prompt are not proof // that protected source structure survived translation. -func validateTranslationValue(key, source, target string) error { +func validateTranslationValue(key, source, target, targetLocale string) error { if strings.TrimSpace(source) != "" && strings.TrimSpace(target) == "" { return fmt.Errorf("blank translation for %q", key) } if findings := validation.ProtectedFindings(key, source, target); len(findings) > 0 { return fmt.Errorf("%s for %q", findings[0].Message, key) } + if findings := validation.ICUFindings(key, source, target, targetLocale); len(findings) > 0 { + return fmt.Errorf("%s for %q", findings[0].Message, key) + } return nil } diff --git a/internal/translate/protected_test.go b/internal/translate/protected_test.go index 4226424..7f535d5 100644 --- a/internal/translate/protected_test.go +++ b/internal/translate/protected_test.go @@ -5,7 +5,7 @@ import "testing" func TestValidateTranslationValuePreservesProtectedMarkdown(t *testing.T) { source := "Read [the guide](https://example.com/guide) and run `go test`.\n\n```go\nfmt.Println(\"ok\")\n```\n" valid := "Lisez [le guide](https://example.com/guide) et lancez `go test`.\n\n```go\nfmt.Println(\"ok\")\n```\n" - if err := validateTranslationValue("_content", source, valid); err != nil { + if err := validateTranslationValue("_content", source, valid, "fr"); err != nil { t.Fatalf("valid protected Markdown was rejected: %v", err) } @@ -16,7 +16,7 @@ func TestValidateTranslationValuePreservesProtectedMarkdown(t *testing.T) { } for name, target := range tests { t.Run(name, func(t *testing.T) { - if err := validateTranslationValue("_content", source, target); err == nil { + if err := validateTranslationValue("_content", source, target, "fr"); err == nil { t.Fatal("invalid protected Markdown was accepted") } }) diff --git a/internal/translate/translate.go b/internal/translate/translate.go index 8db397e..0b100b0 100644 --- a/internal/translate/translate.go +++ b/internal/translate/translate.go @@ -111,15 +111,12 @@ func Run(ctx context.Context, cfg *config.Config, provider llm.Provider, opts Op locales := cfg.TargetLocales if len(opts.Locales) > 0 { - configured := make(map[string]struct{}, len(cfg.TargetLocales)) - for _, locale := range cfg.TargetLocales { - configured[locale] = struct{}{} - } seen := make(map[string]struct{}, len(opts.Locales)) locales = make([]string, 0, len(opts.Locales)) - for _, locale := range opts.Locales { - if _, ok := configured[locale]; !ok { - return nil, fmt.Errorf("locale %q is not in target_locales", locale) + for _, requested := range opts.Locales { + locale, ok := cfg.ConfiguredTargetLocale(requested) + if !ok { + return nil, fmt.Errorf("locale %q is not in target_locales", requested) } if _, ok := seen[locale]; ok { continue @@ -159,7 +156,7 @@ func Run(ctx context.Context, cfg *config.Config, provider llm.Provider, opts Op currentProvider := provider if localeProvider, ok := opts.LocaleProviders[current.locale]; ok { currentProvider = localeProvider - } else if _, hasOverride := cfg.LLM.LocaleOverrides[current.locale]; hasOverride { + } else if cfg.HasLLMOverrideForLocale(current.locale) { currentProvider = nil } outputs[current.index] = translateLocale(ctx, cfg, current.bundle, currentProvider, memory, manifest, current.locale, batchSize, opts) @@ -259,6 +256,14 @@ func translateLocale( } } result := Result{Bundle: bundle.bundle.ID, Locale: locale, KeysTotal: len(sourceKeys)} + for _, key := range sortedKeys(sourceKeys) { + if findings := validation.ICUSourceFindings(key, sourceKeys[key], cfg.SourceLocale); len(findings) > 0 { + result.Errors = append(result.Errors, fmt.Sprintf("source %q: %s", key, findings[0].Message)) + } + } + if len(result.Errors) > 0 { + return jobOutput{result: result} + } targetPath, err := bundle.bundle.TargetPath(locale) if err != nil { result.Errors = append(result.Errors, err.Error()) @@ -320,7 +325,7 @@ func translateLocale( if !plan.state.adoptable() { continue } - if err := validateTranslationValue(plan.key, plan.source, targetKeys[plan.key]); err != nil { + if err := validateTranslationValue(plan.key, plan.source, targetKeys[plan.key], locale); err != nil { result.Errors = append(result.Errors, fmt.Sprintf("cannot adopt %q: %v", plan.key, err)) } } @@ -355,10 +360,12 @@ func translateLocale( toTranslate := make([]plannedEntry, 0, len(candidates)) for _, plan := range candidates { if record, ok := memory.Lookup(locale, bundle.bundle.ID, plan.key, plan.sourceHash, policyHash); ok { - staged[plan.key] = record.Target - origins[plan.key] = translationOrigin{kind: "tm", provider: record.Provider, model: record.Model} - result.KeysCached++ - continue + if err := validateTranslationValue(plan.key, plan.source, record.Target, locale); err == nil { + staged[plan.key] = record.Target + origins[plan.key] = translationOrigin{kind: "tm", provider: record.Provider, model: record.Model} + result.KeysCached++ + continue + } } toTranslate = append(toTranslate, plan) } @@ -390,7 +397,7 @@ func translateLocale( result.Errors = append(result.Errors, fmt.Sprintf("batch %d: provider returned no response", i/batchSize+1)) return jobOutput{result: result} } - if err := validateBatch(entries, response.Translations); err != nil { + if err := validateBatch(entries, response.Translations, locale); err != nil { result.Errors = append(result.Errors, fmt.Sprintf("batch %d: %v", i/batchSize+1, err)) return jobOutput{result: result} } @@ -560,7 +567,7 @@ func (r *Result) addState(state entryState) { } } -func validateBatch(entries []llm.Entry, translations map[string]string) error { +func validateBatch(entries []llm.Entry, translations map[string]string, targetLocale string) error { expected := make(map[string]struct{}, len(entries)) var missing []string for _, entry := range entries { @@ -577,7 +584,7 @@ func validateBatch(entries []llm.Entry, translations map[string]string) error { } if len(missing) == 0 && len(extra) == 0 { for _, entry := range entries { - if err := validateTranslationValue(entry.Key, entry.Value, translations[entry.Key]); err != nil { + if err := validateTranslationValue(entry.Key, entry.Value, translations[entry.Key], targetLocale); err != nil { return err } } diff --git a/internal/translate/translate_test.go b/internal/translate/translate_test.go index 0b02a65..32bd5cf 100644 --- a/internal/translate/translate_test.go +++ b/internal/translate/translate_test.go @@ -6,9 +6,13 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/Tom-R-Main/Internationalizer/internal/config" "github.com/Tom-R-Main/Internationalizer/internal/llm" + "github.com/Tom-R-Main/Internationalizer/internal/policy" + "github.com/Tom-R-Main/Internationalizer/internal/state" + "github.com/Tom-R-Main/Internationalizer/internal/tm" validation "github.com/Tom-R-Main/Internationalizer/internal/validate" ) @@ -54,6 +58,111 @@ func TestRunRejectsIncompleteProviderResponseWithoutWritingTarget(t *testing.T) } } +func TestRunRejectsInvalidICUProviderResponseWithoutWritingTarget(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One item} other {# items}}"}`), 0o644); err != nil { + t.Fatal(err) + } + cfg := testConfig(dir, sourcePath) + provider := &fakeProvider{response: &llm.TranslateResponse{Translations: map[string]string{ + "items": "{count, plural, one {Un article} other {# articles}", + }}} + + results, err := Run(context.Background(), cfg, provider, Options{}) + if err == nil { + t.Fatal("Run returned nil error for malformed ICU provider output") + } + if len(results) != 1 || len(results[0].Errors) == 0 { + t.Fatalf("Run results = %#v, want one locale error", results) + } + if _, statErr := os.Stat(filepath.Join(dir, "fr.json")); !os.IsNotExist(statErr) { + t.Fatalf("target was written after malformed ICU output; stat error = %v", statErr) + } +} + +func TestRunRejectsMalformedICUSourceWithoutCallingProvider(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One}}"}`), 0o644); err != nil { + t.Fatal(err) + } + cfg := testConfig(dir, sourcePath) + provider := &fakeProvider{response: &llm.TranslateResponse{Translations: map[string]string{ + "items": "{count, plural, one {Un} other {Autres}}", + }}} + + results, err := Run(context.Background(), cfg, provider, Options{}) + if err == nil { + t.Fatal("Run returned nil error for malformed ICU source") + } + if provider.calls != 0 { + t.Fatalf("provider called %d times for malformed ICU source", provider.calls) + } + if len(results) != 1 || len(results[0].Errors) == 0 { + t.Fatalf("Run results = %#v, want source error", results) + } +} + +func TestRunDoesNotReuseStructurallyInvalidTMRecord(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + source := "{count, plural, one {One item} other {# items}}" + if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One item} other {# items}}"}`), 0o644); err != nil { + t.Fatal(err) + } + cfg := testConfig(dir, sourcePath) + resolved, err := policy.Resolve(cfg, "fr", "json", "", nil) + if err != nil { + t.Fatal(err) + } + memory, err := tm.Load(cfg.TMPath) + if err != nil { + t.Fatal(err) + } + if err := memory.Add(tm.Record{ + Bundle: "default", + Key: "items", + Source: source, + Target: "{count, plural, one {Un article} other {# articles}", + Locale: "fr", + Hash: state.SourceHash("json", source), + PolicyHash: resolved.Hash, + Timestamp: time.Now(), + }); err != nil { + t.Fatal(err) + } + provider := &fakeProvider{response: &llm.TranslateResponse{Translations: map[string]string{ + "items": "{count, plural, one {Un article} other {# articles}}", + }}} + + results, err := Run(context.Background(), cfg, provider, Options{}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].KeysCached != 0 || results[0].KeysTranslated != 1 || provider.calls != 1 { + t.Fatalf("Run results = %#v, provider calls = %d; want invalid TM bypass and fresh translation", results, provider.calls) + } +} + +func TestRunSelectsCanonicalEquivalentConfiguredLocale(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + if err := os.WriteFile(sourcePath, []byte(`{"save":"Save"}`), 0o644); err != nil { + t.Fatal(err) + } + cfg := testConfig(dir, sourcePath) + cfg.TargetLocales = []string{"pt-br"} + + results, err := Run(context.Background(), cfg, nil, Options{DryRun: true, Locales: []string{"pt-BR"}}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].Locale != "pt-br" || results[0].TargetPath != filepath.Join(dir, "pt-br.json") { + t.Fatalf("Run results = %#v, want configured locale spelling and path", results) + } +} + func TestRunProducesTargetLocalePluralFormsThatPassStrictValidation(t *testing.T) { dir := t.TempDir() sourcePath := filepath.Join(dir, "en.json") diff --git a/internal/validate/findings.go b/internal/validate/findings.go index 1b5d03b..c446101 100644 --- a/internal/validate/findings.go +++ b/internal/validate/findings.go @@ -13,6 +13,9 @@ const ( CodeProtectedStructureMismatch FindingCode = "protected_structure_mismatch" CodeGlossaryViolation FindingCode = "glossary_violation" CodePluralFormMissing FindingCode = "plural_form_missing" + CodeICUMessageSyntax FindingCode = "icu_message_syntax" + CodeICUArgumentMismatch FindingCode = "icu_argument_mismatch" + CodeICUSelectorMismatch FindingCode = "icu_selector_mismatch" CodeUntracked FindingCode = "untracked" CodeSourceStale FindingCode = "source_stale" CodePolicyStale FindingCode = "policy_stale" diff --git a/internal/validate/plurals.go b/internal/validate/plurals.go index 4bb9b45..51d56ba 100644 --- a/internal/validate/plurals.go +++ b/internal/validate/plurals.go @@ -1,66 +1,13 @@ package validate -import "strings" +import ( + "strings" -// CLDRPluralForms maps locale codes to their required CLDR plural categories. -// Source: Unicode CLDR plural rules for the most common languages. -var CLDRPluralForms = map[string][]string{ - // Germanic - "en": {"one", "other"}, - "en-US": {"one", "other"}, - "en-GB": {"one", "other"}, - "de": {"one", "other"}, - "nl": {"one", "other"}, - "sv": {"one", "other"}, - "da": {"one", "other"}, - "nb": {"one", "other"}, - "fi": {"one", "other"}, - - // Romance - "fr": {"one", "many", "other"}, - "fr-CA": {"one", "many", "other"}, - "es": {"one", "many", "other"}, - "es-MX": {"one", "many", "other"}, - "pt": {"one", "many", "other"}, - "pt-BR": {"one", "many", "other"}, - "it": {"one", "many", "other"}, - "ro": {"one", "few", "other"}, - - // Slavic - "ru": {"one", "few", "many", "other"}, - "uk": {"one", "few", "many", "other"}, - "pl": {"one", "few", "many", "other"}, - "cs": {"one", "few", "many", "other"}, - - // Semitic - "ar": {"zero", "one", "two", "few", "many", "other"}, - "he": {"one", "two", "other"}, - - // Indic - "hi": {"one", "other"}, - "bn": {"one", "other"}, - "pa": {"one", "other"}, - "te": {"one", "other"}, - - // CJK (no plural forms — only "other") - "ja": {"other"}, - "ko": {"other"}, - "zh": {"other"}, - "zh-CN": {"other"}, - "zh-TW": {"other"}, - "yue": {"other"}, - - // Other - "tr": {"one", "other"}, - "id": {"other"}, - "vi": {"other"}, - "th": {"other"}, - "el": {"one", "other"}, - "hu": {"one", "other"}, -} + localeid "github.com/Tom-R-Main/Internationalizer/internal/locale" +) // PluralFormsFor returns the CLDR plural categories for a locale. -// Falls back to ["one", "other"] if the locale is not in the table. +// It retains the historical ["one", "other"] fallback for an invalid tag. func PluralFormsFor(locale string) []string { if forms, ok := KnownPluralFormsFor(locale); ok { return forms @@ -68,22 +15,11 @@ func PluralFormsFor(locale string) []string { return []string{"one", "other"} } -// KnownPluralFormsFor returns configured CLDR categories without guessing for -// unknown locales. Strict validation uses this form to avoid false failures. +// KnownPluralFormsFor returns CLDR-backed cardinal categories for a valid +// locale. Invalid or unrecognized tags are not guessed. func KnownPluralFormsFor(locale string) ([]string, bool) { - if forms, ok := CLDRPluralForms[locale]; ok { - return forms, true - } - for configuredLocale, forms := range CLDRPluralForms { - if strings.EqualFold(configuredLocale, locale) { - return forms, true - } - } - if separator := strings.IndexAny(locale, "-_"); separator > 0 { - forms, ok := CLDRPluralForms[strings.ToLower(locale[:separator])] - return forms, ok - } - return nil, false + forms, err := localeid.CardinalCategories(locale) + return forms, err == nil } // ExpandI18nextV4Source returns the source key set plus any target-only plural diff --git a/internal/validate/strict_test.go b/internal/validate/strict_test.go index cec7633..1369f3c 100644 --- a/internal/validate/strict_test.go +++ b/internal/validate/strict_test.go @@ -221,14 +221,14 @@ func TestTargetOnlyPluralFormsReceiveContentAndStateValidation(t *testing.T) { } } -func TestUnknownLocaleDoesNotGuessPluralRequirements(t *testing.T) { +func TestPrivateUseLocaleUsesRootPluralRequirements(t *testing.T) { cfg := validationConfig(t, map[string]string{"items_one": "{{count}} item", "items_other": "{{count}} items"}, map[string]string{"items_one": "{{count}} x", "items_other": "{{count}} xs"}, ) oldTarget := targetPath(cfg) - cfg.TargetLocales = []string{"xx-ZZ"} - newTarget := filepath.Join(filepath.Dir(oldTarget), "xx-ZZ.json") + cfg.TargetLocales = []string{"qaa-ZZ"} + newTarget := filepath.Join(filepath.Dir(oldTarget), "qaa-ZZ.json") if err := os.Rename(oldTarget, newTarget); err != nil { t.Fatal(err) } @@ -239,7 +239,7 @@ func TestUnknownLocaleDoesNotGuessPluralRequirements(t *testing.T) { t.Fatal(err) } if findingByCode(reports[0], CodePluralFormMissing).Code != "" { - t.Fatalf("unknown locale received guessed plural finding: %#v", reports[0]) + t.Fatalf("private-use locale received an unsupported plural finding: %#v", reports[0]) } writeJSON(t, newTarget, map[string]string{"items_other": "{{count}} xs"}) @@ -247,7 +247,9 @@ func TestUnknownLocaleDoesNotGuessPluralRequirements(t *testing.T) { if err != nil { t.Fatal(err) } - assertFindingCodes(t, reports[0], CodeMissingKey) + if HasFailures(reports) { + t.Fatalf("root plural fallback rejected an other-only target: %#v", reports[0]) + } } func TestI18nextPluralCoverageUsesTargetLocaleRequirements(t *testing.T) { diff --git a/internal/validate/validate.go b/internal/validate/validate.go index ccdc35f..38bfb77 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -11,6 +11,7 @@ import ( "github.com/Tom-R-Main/Internationalizer/internal/config" "github.com/Tom-R-Main/Internationalizer/internal/formats" "github.com/Tom-R-Main/Internationalizer/internal/glossary" + "github.com/Tom-R-Main/Internationalizer/internal/message" "github.com/Tom-R-Main/Internationalizer/internal/policy" "github.com/Tom-R-Main/Internationalizer/internal/state" "github.com/Tom-R-Main/Internationalizer/internal/styleguide" @@ -139,6 +140,12 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s if (opts.Strict || opts.RequireState) && pluralStyle == "i18next-v4" { validationKeys, requiredPluralKeys, optionalPluralKeys = ExpandI18nextV4Source(sourceKeys, sourceLocale, locale) } + sourceICUValid := make(map[string]bool, len(validationKeys)) + for key, sourceValue := range validationKeys { + findings := ICUSourceFindings(key, sourceValue, sourceLocale) + sourceICUValid[key] = len(findings) == 0 + report.Findings = append(report.Findings, findings...) + } targetData, err := os.ReadFile(targetPath) if err != nil { @@ -197,6 +204,9 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s if mismatch := InterpolationMismatch(key, sourceValue, targetValue); mismatch != nil { report.Mismatches = append(report.Mismatches, *mismatch) } + if sourceICUValid[key] { + report.Findings = append(report.Findings, ICUFindings(key, sourceValue, targetValue, locale)...) + } if opts.Strict { report.Findings = append(report.Findings, ProtectedFindings(key, sourceValue, targetValue)...) report.Findings = append(report.Findings, glossaryFindings(key, sourceValue, targetValue, terms)...) @@ -235,6 +245,38 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s return report } +// ICUFindings compares ICU MessageFormat structure independently of linguistic +// strictness. Message syntax and branch identity are runtime correctness. +func ICUFindings(key, source, target, targetLocale string) []Finding { + return messageFindings(key, message.Compare(source, target, targetLocale)) +} + +// ICUSourceFindings validates source syntax and source-locale plural selectors +// even when a target file does not exist yet. +func ICUSourceFindings(key, source, sourceLocale string) []Finding { + return messageFindings(key, message.Compare(source, source, sourceLocale)) +} + +func messageFindings(key string, issues []message.Issue) []Finding { + findings := make([]Finding, 0, len(issues)) + for _, issue := range issues { + code := CodeICUArgumentMismatch + switch issue.Code { + case message.CodeSyntax: + code = CodeICUMessageSyntax + case message.CodeSelectorMismatch, message.CodeInvalidPluralCategory: + code = CodeICUSelectorMismatch + } + findings = append(findings, Finding{ + Code: code, + Severity: SeverityError, + Key: key, + Message: issue.Message, + }) + } + return findings +} + func missingFinding(key string, requiredPluralKeys map[string]struct{}) Finding { if _, required := requiredPluralKeys[key]; required { return Finding{Code: CodePluralFormMissing, Severity: SeverityError, Key: key, Message: "target locale requires this plural form"} diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go index 2b5cbad..996aa49 100644 --- a/internal/validate/validate_test.go +++ b/internal/validate/validate_test.go @@ -3,6 +3,7 @@ package validate import ( "os" "path/filepath" + "strconv" "strings" "testing" @@ -83,3 +84,99 @@ func TestFormatHumanPreservesInterpolationMismatchDetails(t *testing.T) { t.Fatalf("FormatHuman omitted mismatch details: %q", got) } } + +func TestValidateReportsICUStructureFailures(t *testing.T) { + tests := []struct { + name string + locale string + source string + target string + code FindingCode + }{ + { + name: "malformed target", + locale: "fr", + source: `{count, plural, one {One item} other {# items}}`, + target: `{count, plural, one {Un article} other {# articles}`, + code: CodeICUMessageSyntax, + }, + { + name: "argument type", + locale: "fr", + source: `{count, plural, one {One item} other {# items}}`, + target: `{count, select, one {Un article} other {Des articles}}`, + code: CodeICUArgumentMismatch, + }, + { + name: "locale category", + locale: "te", + source: `{count, plural, one {One item} other {# items}}`, + target: `{count, plural, one {ఒక అంశం} few {కొన్ని అంశాలు} other {# అంశాలు}}`, + code: CodeICUSelectorMismatch, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + report := validateMessagePair(t, test.locale, test.source, test.target) + if !reportHasFinding(report, test.code) || !HasFailures([]Report{report}) { + t.Fatalf("report = %#v, want failing finding %q", report, test.code) + } + }) + } +} + +func TestValidateAllowsLocalePluralBranchesAndExactSelectors(t *testing.T) { + report := validateMessagePair(t, "ru", + `{count, plural, one {One item} other {# items}}`, + `{count, plural, =0 {Нет элементов} one {Один элемент} few {# элемента} other {# элементов}}`, + ) + if HasFailures([]Report{report}) { + t.Fatalf("valid Russian ICU message reported as failure: %#v", report) + } +} + +func TestValidateReportsMalformedICUSourceWithoutTarget(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One}}"}`), 0o644); err != nil { + t.Fatal(err) + } + reports, err := Validate(&config.Config{SourceLocale: "en", TargetLocales: []string{"fr"}, SourcePath: sourcePath}) + if err != nil { + t.Fatal(err) + } + if len(reports) != 1 || !reportHasFinding(reports[0], CodeICUMessageSyntax) { + t.Fatalf("reports = %#v, want source syntax finding", reports) + } +} + +func validateMessagePair(t *testing.T, locale, source, target string) Report { + t.Helper() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + targetPath := filepath.Join(dir, locale+".json") + if err := os.WriteFile(sourcePath, []byte(`{"items":`+strconv.Quote(source)+`}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(targetPath, []byte(`{"items":`+strconv.Quote(target)+`}`), 0o644); err != nil { + t.Fatal(err) + } + reports, err := Validate(&config.Config{SourceLocale: "en", TargetLocales: []string{locale}, SourcePath: sourcePath}) + if err != nil { + t.Fatal(err) + } + if len(reports) != 1 { + t.Fatalf("Validate returned %d reports, want 1", len(reports)) + } + return reports[0] +} + +func reportHasFinding(report Report, code FindingCode) bool { + for _, finding := range report.Findings { + if finding.Code == code { + return true + } + } + return false +} From 38194727910ddf240bb0672960738140fff92e8b Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 19:54:30 -0400 Subject: [PATCH 02/10] fix: preserve nested ICU context Carry enclosing plural context through nested select branches and accept Unicode ICU argument identifiers. Add regressions for both write-boundary validation gaps. Signed-off-by: Tom Main --- internal/message/message.go | 89 ++++++++++++++++++++++++-------- internal/message/message_test.go | 29 +++++++++++ 2 files changed, 96 insertions(+), 22 deletions(-) diff --git a/internal/message/message.go b/internal/message/message.go index 9a5496f..d757ced 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "unicode" + "unicode/utf8" localeid "github.com/Tom-R-Main/Internationalizer/internal/locale" ) @@ -101,18 +102,30 @@ func LooksLike(input string) bool { continue } cursor := index + 1 - for cursor < len(input) && isSpace(input[cursor]) { - cursor++ + for cursor < len(input) { + width := spaceWidth(input, cursor) + if width == 0 { + break + } + cursor += width } start := cursor - for cursor < len(input) && isIdentifierByte(input[cursor]) { - cursor++ + for cursor < len(input) { + width := identifierWidth(input, cursor) + if width == 0 { + break + } + cursor += width } if cursor == start { continue } - for cursor < len(input) && isSpace(input[cursor]) { - cursor++ + for cursor < len(input) { + width := spaceWidth(input, cursor) + if width == 0 { + break + } + cursor += width } if cursor >= len(input) || input[cursor] == '}' || input[cursor] == ',' { return true @@ -416,7 +429,7 @@ func (p *parser) parseMessage(nested, pluralContext bool) (*Message, error) { switch current := p.input[p.position]; current { case '{': flush() - argument, err := p.parseArgument() + argument, err := p.parseArgument(pluralContext) if err != nil { return nil, err } @@ -492,7 +505,7 @@ func (p *parser) apostropheStartsQuote(pluralContext bool) bool { } } -func (p *parser) parseArgument() (*Argument, error) { +func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { p.position++ p.skipSpace() name := p.readIdentifier() @@ -575,7 +588,8 @@ func (p *parser) parseArgument() (*Argument, error) { if !p.consume('{') { return nil, p.errorf("expected message branch for selector %q", selector) } - branch, err := p.parseBranch(argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal) + branchContext := pluralContext || argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal + branch, err := p.parseBranch(branchContext) if err != nil { return nil, err } @@ -596,8 +610,12 @@ func (p *parser) parseArgument() (*Argument, error) { func (p *parser) readIdentifier() string { start := p.position - for p.position < len(p.input) && isIdentifierByte(p.input[p.position]) { - p.position++ + for p.position < len(p.input) { + width := identifierWidth(p.input, p.position) + if width == 0 { + break + } + p.position += width } return p.input[start:p.position] } @@ -605,15 +623,23 @@ func (p *parser) readIdentifier() string { func (p *parser) readSelector() string { p.skipSpace() start := p.position - for p.position < len(p.input) && !isSpace(p.input[p.position]) && p.input[p.position] != '{' && p.input[p.position] != '}' { - p.position++ + for p.position < len(p.input) && p.input[p.position] != '{' && p.input[p.position] != '}' { + if spaceWidth(p.input, p.position) != 0 { + break + } + _, width := utf8.DecodeRuneInString(p.input[p.position:]) + p.position += width } return p.input[start:p.position] } func (p *parser) skipSpace() { - for p.position < len(p.input) && isSpace(p.input[p.position]) { - p.position++ + for p.position < len(p.input) { + width := spaceWidth(p.input, p.position) + if width == 0 { + break + } + p.position += width } } @@ -629,12 +655,30 @@ func (p *parser) errorf(format string, arguments ...any) error { return fmt.Errorf("at byte %d: %s", p.position, fmt.Sprintf(format, arguments...)) } -func isIdentifierByte(value byte) bool { - return value == '_' || value == '-' || value == '.' || value >= '0' && value <= '9' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' +func identifierWidth(input string, position int) int { + character, width := utf8.DecodeRuneInString(input[position:]) + if character == utf8.RuneError && width == 1 { + return 0 + } + if character < utf8.RuneSelf { + value := byte(character) + if value == '_' || value == '-' || value == '.' || value >= '0' && value <= '9' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' { + return width + } + return 0 + } + if unicode.Is(unicode.Pattern_Syntax, character) || unicode.Is(unicode.Pattern_White_Space, character) { + return 0 + } + return width } -func isSpace(value byte) bool { - return unicode.IsSpace(rune(value)) +func spaceWidth(input string, position int) int { + character, width := utf8.DecodeRuneInString(input[position:]) + if character == utf8.RuneError && width == 1 || !unicode.IsSpace(character) { + return 0 + } + return width } func isPluralSelector(selector string) bool { @@ -664,12 +708,12 @@ func (message *Message) writeTo(output *strings.Builder, pluralContext bool) { case elementPound: output.WriteByte('#') case elementArgument: - element.argument.writeTo(output) + element.argument.writeTo(output, pluralContext) } } } -func (argument *Argument) writeTo(output *strings.Builder) { +func (argument *Argument) writeTo(output *strings.Builder, pluralContext bool) { output.WriteByte('{') output.WriteString(argument.Name) if argument.Type == ArgumentSimple { @@ -698,7 +742,8 @@ func (argument *Argument) writeTo(output *strings.Builder) { } output.WriteString(option.Selector) output.WriteString(" {") - option.Message.writeTo(output, argument.Type == ArgumentPlural || argument.Type == ArgumentSelectOrdinal) + branchContext := pluralContext || argument.Type == ArgumentPlural || argument.Type == ArgumentSelectOrdinal + option.Message.writeTo(output, branchContext) output.WriteByte('}') } output.WriteByte('}') diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 1ffa332..5dd46de 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -171,6 +171,35 @@ func TestComparePreservesPoundPlaceholders(t *testing.T) { } } +func TestComparePreservesPluralPoundThroughNestedSelect(t *testing.T) { + source := "{n, plural, other {{gender, select, male {# men} other {# people}}}}" + target := "{n, plural, other {{gender, select, male {hommes} other {# personnes}}}}" + issues := Compare(source, target, "fr") + if !hasCode(issues, CodeSelectorMismatch) { + t.Fatalf("Compare issues = %#v, want nested missing pound placeholder", issues) + } +} + +func TestCompareSupportsUnicodeArgumentNames(t *testing.T) { + source := "Bienvenue, {имя}" + target := "Bienvenue" + if !LooksLike(source) { + t.Fatalf("LooksLike(%q) = false", source) + } + issues := Compare(source, target, "fr") + if !hasCode(issues, CodeArgumentMismatch) { + t.Fatalf("Compare issues = %#v, want missing Unicode argument", issues) + } + + parsed, err := Parse("{имя, select, admin {Привет, {имя}} other {Здравствуйте, {имя}}}") + if err != nil { + t.Fatal(err) + } + if _, err := Parse(parsed.String()); err != nil { + t.Fatalf("Parse(String()) failed: %v", err) + } +} + func TestCompareReportsMalformedSourceAndTarget(t *testing.T) { for name, values := range map[string][2]string{ "source": {"{count, plural, one {One}}", "{count, plural, one {Un} other {Autres}}"}, From f81a9699b86e558e87e4772e5d831b25d8fea887 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:01:13 -0400 Subject: [PATCH 03/10] fix: tighten plural branch validation Reject unsupported arguments in target-only exact branches, enforce finite ICU exact-selector syntax, and add modern Ladin plural categories with regressions. Signed-off-by: Tom Main --- internal/locale/locale.go | 6 +-- internal/locale/locale_test.go | 1 + internal/message/message.go | 75 ++++++++++++++++++++++++++++++-- internal/message/message_test.go | 28 ++++++++++++ 4 files changed, 104 insertions(+), 6 deletions(-) diff --git a/internal/locale/locale.go b/internal/locale/locale.go index bdfe712..00e9807 100644 --- a/internal/locale/locale.go +++ b/internal/locale/locale.go @@ -155,7 +155,7 @@ func categories(input string, pluralType PluralType) ([]string, error) { func hasCompactExponentMany(tag language.Tag) bool { base, _ := tag.Base() switch base.String() { - case "ca", "es", "fr", "gl", "it", "pt", "scn", "vec": + case "ca", "es", "fr", "gl", "it", "lld", "pt", "scn", "vec": return true default: return false @@ -179,7 +179,7 @@ func needsCurrentCardinalOne(tag language.Tag) bool { base, _ := tag.Base() // These languages postdate x/text's embedded plural tables. switch base.String() { - case "scn", "vec": + case "lld", "scn", "vec": return true default: return false @@ -196,7 +196,7 @@ func needsCurrentOrdinalMany(tag language.Tag) bool { base, _ := tag.Base() // Sicilian and Venetian ordinal rules postdate x/text's embedded tables. switch base.String() { - case "kw", "scn", "vec": + case "kw", "lld", "scn", "vec": return true default: return false diff --git a/internal/locale/locale_test.go b/internal/locale/locale_test.go index 5fa51ac..91072cf 100644 --- a/internal/locale/locale_test.go +++ b/internal/locale/locale_test.go @@ -86,6 +86,7 @@ func TestSupportedLocalesMatchCLDR49CategorySets(t *testing.T) { "ja": {[]string{"other"}, []string{"other"}}, "ko": {[]string{"other"}, []string{"other"}}, "kw": {[]string{"zero", "one", "two", "few", "many", "other"}, []string{"one", "many", "other"}}, + "lld": {[]string{"one", "many", "other"}, []string{"many", "other"}}, "ms": {[]string{"other"}, []string{"one", "other"}}, "mt": {[]string{"one", "two", "few", "many", "other"}, []string{"other"}}, "nl": {[]string{"one", "other"}, []string{"other"}}, diff --git a/internal/message/message.go b/internal/message/message.go index d757ced..edac1be 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -300,7 +300,12 @@ func compareArguments(source, target *Argument, targetLocale string) []Issue { } sourceOther := sourceOptions["other"] for selector, targetBranch := range targetOptions { - if _, sourceHasSelector := sourceOptions[selector]; !sourceHasSelector && !isExactSelector(selector) { + if _, sourceHasSelector := sourceOptions[selector]; sourceHasSelector { + continue + } + if isExactSelector(selector) { + issues = append(issues, compareTargetArgumentSubset(sourceOther, targetBranch, targetLocale)...) + } else { issues = append(issues, compareMessages(sourceOther, targetBranch, targetLocale, name)...) } } @@ -308,6 +313,50 @@ func compareArguments(source, target *Argument, targetLocale string) []Issue { return issues } +func compareTargetArgumentSubset(source, target *Message, targetLocale string) []Issue { + sourceArguments := immediateArguments(source) + targetArguments := immediateArguments(target) + var issues []Issue + for name, targetOccurrences := range targetArguments { + sourceOccurrences, ok := sourceArguments[name] + if !ok { + issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU exact-selector branch introduces argument %q that is absent from the source fallback branch", name)}) + continue + } + if len(targetOccurrences) > len(sourceOccurrences) || !hasArgumentSubsetMatching(sourceOccurrences, targetOccurrences, targetLocale) { + issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU exact-selector branch introduces an incompatible occurrence of argument %q", name)}) + } + } + return issues +} + +func hasArgumentSubsetMatching(source, target []*Argument, targetLocale string) bool { + matchedTarget := make([]int, len(source)) + for index := range matchedTarget { + matchedTarget[index] = -1 + } + var match func(int, []bool) bool + match = func(targetIndex int, visited []bool) bool { + for sourceIndex := range source { + if visited[sourceIndex] || len(compareArguments(source[sourceIndex], target[targetIndex], targetLocale)) != 0 { + continue + } + visited[sourceIndex] = true + if matchedTarget[sourceIndex] == -1 || match(matchedTarget[sourceIndex], visited) { + matchedTarget[sourceIndex] = targetIndex + return true + } + } + return false + } + for targetIndex := range target { + if !match(targetIndex, make([]bool, len(source))) { + return false + } + } + return true +} + func comparePluralSelectors(source, target *Argument, targetLocale string) []Issue { name := source.Name categories, err := localeid.CardinalCategories(targetLocale) @@ -395,8 +444,28 @@ func isExactSelector(selector string) bool { if !strings.HasPrefix(selector, "=") || len(selector) == 1 { return false } - _, err := strconv.ParseFloat(selector[1:], 64) - return err == nil + number := selector[1:] + if number[0] == '-' { + number = number[1:] + } + if number == "" { + return false + } + dotSeen := false + digitSeenAfterDot := false + for _, character := range number { + switch { + case character >= '0' && character <= '9': + if dotSeen { + digitSeenAfterDot = true + } + case character == '.' && !dotSeen: + dotSeen = true + default: + return false + } + } + return !dotSeen || digitSeenAfterDot } type parser struct { diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 5dd46de..f40b69c 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -83,6 +83,34 @@ func TestCompareAllowsLocaleCategoryAndExactSelector(t *testing.T) { } } +func TestCompareRejectsArgumentIntroducedByTargetOnlyExactSelector(t *testing.T) { + issues := Compare( + "{n, plural, other {# items}}", + "{n, plural, =0 {{price, number}} other {# articles}}", + "fr", + ) + if !hasCode(issues, CodeArgumentMismatch) { + t.Fatalf("Compare issues = %#v, want target-only exact argument mismatch", issues) + } +} + +func TestParseRejectsNonFiniteAndMalformedExactSelectors(t *testing.T) { + for _, selector := range []string{"NaN", "Inf", "Infinity", "1.", "0x1p2"} { + t.Run(selector, func(t *testing.T) { + if _, err := Parse("{n, plural, =" + selector + " {bad} other {ok}}"); err == nil { + t.Fatalf("Parse accepted invalid exact selector %q", selector) + } + }) + } + for _, selector := range []string{"0", "-1", "1.5"} { + t.Run("valid_"+selector, func(t *testing.T) { + if _, err := Parse("{n, plural, =" + selector + " {ok} other {other}}"); err != nil { + t.Fatalf("Parse rejected valid exact selector %q: %v", selector, err) + } + }) + } +} + func TestCompareChecksTargetOnlyLocaleCategoryAgainstSourceOther(t *testing.T) { source := "{count, plural, one {{name} has one item} other {{name} has # items}}" valid := "{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} other {{name} имеет # предметов}}" From 2631b48e28c62850206e260336a28517be4ed5ca Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:05:46 -0400 Subject: [PATCH 04/10] fix: align ICU quote and selector semantics Treat end-of-pattern as an implicit apostrophe quote close and canonicalize exact numeric selectors before structural comparison. Signed-off-by: Tom Main --- internal/message/message.go | 44 ++++++++++++++++++++++++++++---- internal/message/message_test.go | 25 ++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/internal/message/message.go b/internal/message/message.go index edac1be..f856cb1 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -441,15 +441,22 @@ func displayType(argumentType ArgumentType) string { } func isExactSelector(selector string) bool { + _, ok := normalizeExactSelector(selector) + return ok +} + +func normalizeExactSelector(selector string) (string, bool) { if !strings.HasPrefix(selector, "=") || len(selector) == 1 { - return false + return "", false } number := selector[1:] + negative := false if number[0] == '-' { + negative = true number = number[1:] } if number == "" { - return false + return "", false } dotSeen := false digitSeenAfterDot := false @@ -462,10 +469,30 @@ func isExactSelector(selector string) bool { case character == '.' && !dotSeen: dotSeen = true default: - return false + return "", false } } - return !dotSeen || digitSeenAfterDot + if dotSeen && !digitSeenAfterDot { + return "", false + } + integer, fraction, _ := strings.Cut(number, ".") + integer = strings.TrimLeft(integer, "0") + if integer == "" { + integer = "0" + } + fraction = strings.TrimRight(fraction, "0") + if integer == "0" && fraction == "" { + negative = false + } + canonical := "=" + if negative { + canonical += "-" + } + canonical += integer + if fraction != "" { + canonical += "." + fraction + } + return canonical, true } type parser struct { @@ -560,7 +587,9 @@ func (p *parser) parseApostrophe(pluralContext bool) (string, error) { p.position++ return quoted.String(), nil } - return "", p.errorf("unterminated apostrophe quote") + // ICU MessageFormat treats the end of a pattern as the implicit end of an + // apostrophe-quoted syntax span. + return quoted.String(), nil } func (p *parser) apostropheStartsQuote(pluralContext bool) bool { @@ -646,6 +675,11 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { if selector == "" { return nil, p.errorf("selector is required for argument %q", name) } + if argumentType != ArgumentSelect { + if canonical, ok := normalizeExactSelector(selector); ok { + selector = canonical + } + } if _, duplicate := selectors[selector]; duplicate { return nil, p.errorf("duplicate selector %q for argument %q", selector, name) } diff --git a/internal/message/message_test.go b/internal/message/message_test.go index f40b69c..185747f 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -111,6 +111,31 @@ func TestParseRejectsNonFiniteAndMalformedExactSelectors(t *testing.T) { } } +func TestCompareNormalizesEquivalentExactSelectors(t *testing.T) { + source := "{n, plural, =1 {One} other {# items}}" + for _, selector := range []string{"1.0", "01", "01.000"} { + t.Run(selector, func(t *testing.T) { + target := "{n, plural, =" + selector + " {Un} other {# articles}}" + if issues := Compare(source, target, "fr"); len(issues) != 0 { + t.Fatalf("Compare rejected equivalent selector %q: %#v", selector, issues) + } + }) + } + if _, err := Parse("{n, plural, =1 {One} =1.0 {Duplicate} other {Other}}"); err == nil { + t.Fatal("Parse accepted numerically duplicate exact selectors") + } +} + +func TestParseImplicitlyClosesApostropheQuoteAtEnd(t *testing.T) { + parsed, err := Parse("Hello {name}: '{") + if err != nil { + t.Fatal(err) + } + if _, err := Parse(parsed.String()); err != nil { + t.Fatalf("Parse(String()) failed: %v", err) + } +} + func TestCompareChecksTargetOnlyLocaleCategoryAgainstSourceOther(t *testing.T) { source := "{count, plural, one {{name} has one item} other {{name} has # items}}" valid := "{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} other {{name} имеет # предметов}}" From d660b71d9f91336d52a61392fe4828b7dfdff6f6 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:11:36 -0400 Subject: [PATCH 05/10] fix: accept ICU offset spacing Allow ICU whitespace after plural offset colons and validate select keywords against identifier boundaries. Signed-off-by: Tom Main --- internal/message/message.go | 21 ++++++++++++++++++++- internal/message/message_test.go | 15 +++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/internal/message/message.go b/internal/message/message.go index f856cb1..fd7bd7c 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -653,6 +653,7 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { if argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal { if strings.HasPrefix(p.input[p.position:], "offset:") { p.position += len("offset:") + p.skipSpace() start := p.position for p.position < len(p.input) && p.input[p.position] >= '0' && p.input[p.position] <= '9' { p.position++ @@ -683,7 +684,11 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { if _, duplicate := selectors[selector]; duplicate { return nil, p.errorf("duplicate selector %q for argument %q", selector, name) } - if argumentType != ArgumentSelect && !isPluralSelector(selector) { + if argumentType == ArgumentSelect { + if !isIdentifier(selector) { + return nil, p.errorf("invalid select keyword %q", selector) + } + } else if !isPluralSelector(selector) { return nil, p.errorf("invalid plural selector %q", selector) } selectors[selector] = struct{}{} @@ -776,6 +781,20 @@ func identifierWidth(input string, position int) int { return width } +func isIdentifier(input string) bool { + if input == "" { + return false + } + for position := 0; position < len(input); { + width := identifierWidth(input, position) + if width == 0 { + return false + } + position += width + } + return true +} + func spaceWidth(input string, position int) int { character, width := utf8.DecodeRuneInString(input[position:]) if character == utf8.RuneError && width == 1 || !unicode.IsSpace(character) { diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 185747f..4c0c25b 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -136,6 +136,21 @@ func TestParseImplicitlyClosesApostropheQuoteAtEnd(t *testing.T) { } } +func TestParseAllowsWhitespaceAfterPluralOffsetColon(t *testing.T) { + if _, err := Parse("{n, plural, offset: 1 one {One} other {# items}}"); err != nil { + t.Fatal(err) + } +} + +func TestParseValidatesSelectKeywords(t *testing.T) { + if _, err := Parse("{g, select, male,female {x} other {y}}"); err == nil { + t.Fatal("Parse accepted punctuation in select keyword") + } + if _, err := Parse("{g, select, мужчина {x} other {y}}"); err != nil { + t.Fatalf("Parse rejected Unicode select keyword: %v", err) + } +} + func TestCompareChecksTargetOnlyLocaleCategoryAgainstSourceOther(t *testing.T) { source := "{count, plural, one {{name} has one item} other {{name} has # items}}" valid := "{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} other {{name} имеет # предметов}}" From fede1909edd2cce1332a675f79c36da3b10fbde8 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:25:20 -0400 Subject: [PATCH 06/10] fix: align ICU validation with runtime grammar Signed-off-by: Tom Main --- internal/formats/fuzz_benchmark_test.go | 1 + internal/formats/json.go | 6 ++++ internal/formats/json_test.go | 16 ++++++++++ internal/message/message.go | 41 ++++++++++++++++--------- internal/message/message_test.go | 37 ++++++++++++++++++++++ internal/translate/protected_test.go | 8 +++++ internal/validate/protected.go | 19 ++++++++++-- internal/validate/validate.go | 11 +++++-- internal/validate/validate_test.go | 29 +++++++++++++++++ 9 files changed, 148 insertions(+), 20 deletions(-) diff --git a/internal/formats/fuzz_benchmark_test.go b/internal/formats/fuzz_benchmark_test.go index 829f5c6..4f44378 100644 --- a/internal/formats/fuzz_benchmark_test.go +++ b/internal/formats/fuzz_benchmark_test.go @@ -13,6 +13,7 @@ func FuzzJSONParseSerializeRoundTrip(f *testing.F) { []byte(`{"items":["One","Two"],"enabled":true,"count":2,"empty":null}`), []byte(`{"numeric":{"0":"Zero"}}`), []byte(`{"nested":{"welcome":"Hello, {name}"}}`), + []byte(`"Save"`), []byte(`not JSON`), } { f.Add(seed) diff --git a/internal/formats/json.go b/internal/formats/json.go index 1049f7f..9ab2b66 100644 --- a/internal/formats/json.go +++ b/internal/formats/json.go @@ -103,6 +103,12 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt return nil, fmt.Errorf("json parse original: %w", err) } replaced := make(map[string]struct{}, len(entries)) + if _, rootString := raw.(string); rootString { + if replacement, ok := entries[""]; ok { + raw = replacement + replaced[""] = struct{}{} + } + } replaceLeaves("", raw, entries, replaced) for key, value := range entries { if _, ok := replaced[key]; ok { diff --git a/internal/formats/json_test.go b/internal/formats/json_test.go index 6ff0c20..557dd42 100644 --- a/internal/formats/json_test.go +++ b/internal/formats/json_test.go @@ -68,6 +68,22 @@ func TestJSONParseExcludesNonStringLeaves(t *testing.T) { } } +func TestJSONRoundTripsRootString(t *testing.T) { + f := &JSONFormat{} + entries, err := f.Parse([]byte(`"Save"`)) + if err != nil { + t.Fatal(err) + } + entries[""] = "Enregistrer" + output, err := f.Serialize(entries, []byte(`"Save"`)) + if err != nil { + t.Fatal(err) + } + if string(output) != `"Enregistrer"` { + t.Fatalf("Serialize() = %s, want translated root string", output) + } +} + func TestJSONSerializePreservesOrder(t *testing.T) { original := `{ "b": "B", diff --git a/internal/message/message.go b/internal/message/message.go index fd7bd7c..7897d58 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -4,7 +4,6 @@ package message import ( "fmt" "sort" - "strconv" "strings" "unicode" "unicode/utf8" @@ -71,7 +70,7 @@ type Argument struct { Name string Type ArgumentType Style string - Offset int + Offset string Options []Option } @@ -237,7 +236,7 @@ func argumentSortKey(argument *Argument) string { options = append(options, option.Selector+"{"+messageSortKey(option.Message)+"}") } sort.Strings(options) - return fmt.Sprintf("%s\x00%s\x00%020d\x00%s", argument.Type, argument.Style, argument.Offset, strings.Join(options, "\x00")) + return fmt.Sprintf("%s\x00%s\x00%s\x00%s", argument.Type, argument.Style, argument.Offset, strings.Join(options, "\x00")) } func messageSortKey(message *Message) string { @@ -449,9 +448,22 @@ func normalizeExactSelector(selector string) (string, bool) { if !strings.HasPrefix(selector, "=") || len(selector) == 1 { return "", false } - number := selector[1:] + canonical, ok := normalizeNumber(selector[1:], true) + if !ok { + return "", false + } + return "=" + canonical, true +} + +func normalizeNumber(number string, allowNegative bool) (string, bool) { + if number == "" { + return "", false + } negative := false if number[0] == '-' { + if !allowNegative { + return "", false + } negative = true number = number[1:] } @@ -484,7 +496,7 @@ func normalizeExactSelector(selector string) (string, bool) { if integer == "0" && fraction == "" { negative = false } - canonical := "=" + canonical := "" if negative { canonical += "-" } @@ -655,17 +667,16 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { p.position += len("offset:") p.skipSpace() start := p.position - for p.position < len(p.input) && p.input[p.position] >= '0' && p.input[p.position] <= '9' { + for p.position < len(p.input) && (p.input[p.position] >= '0' && p.input[p.position] <= '9' || p.input[p.position] == '.') { p.position++ } - if start == p.position { - return nil, p.errorf("plural offset must be a non-negative integer") + offset, ok := normalizeNumber(p.input[start:p.position], false) + if !ok { + return nil, p.errorf("plural offset must be a non-negative finite number") } - offset, err := strconv.Atoi(p.input[start:p.position]) - if err != nil { - return nil, p.errorf("invalid plural offset") + if offset != "0" { + argument.Offset = offset } - argument.Offset = offset p.skipSpace() } } @@ -797,7 +808,7 @@ func isIdentifier(input string) bool { func spaceWidth(input string, position int) int { character, width := utf8.DecodeRuneInString(input[position:]) - if character == utf8.RuneError && width == 1 || !unicode.IsSpace(character) { + if character == utf8.RuneError && width == 1 || !unicode.Is(unicode.Pattern_White_Space, character) { return 0 } return width @@ -853,8 +864,8 @@ func (argument *Argument) writeTo(output *strings.Builder, pluralContext bool) { return } output.WriteString(", ") - if argument.Offset != 0 { - _, _ = fmt.Fprintf(output, "offset:%d ", argument.Offset) + if argument.Offset != "" { + _, _ = fmt.Fprintf(output, "offset:%s ", argument.Offset) } options := append([]Option(nil), argument.Options...) sort.SliceStable(options, func(left, right int) bool { return options[left].Selector < options[right].Selector }) diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 4c0c25b..1a160eb 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -142,6 +142,43 @@ func TestParseAllowsWhitespaceAfterPluralOffsetColon(t *testing.T) { } } +func TestParsePreservesDecimalPluralOffsets(t *testing.T) { + parsed, err := Parse("{n, plural, offset:00.500 one {One} other {# items}}") + if err != nil { + t.Fatal(err) + } + if got := parsed.String(); !strings.Contains(got, "offset:0.5 ") { + t.Fatalf("String() = %q, want canonical decimal offset", got) + } + if issues := Compare( + "{n, plural, offset:0.5 one {One} other {# items}}", + "{n, plural, offset:00.500 one {Un} other {# articles}}", + "fr", + ); len(issues) != 0 { + t.Fatalf("Compare rejected equivalent decimal offsets: %#v", issues) + } + if issues := Compare( + "{n, plural, offset:0.5 one {One} other {# items}}", + "{n, plural, offset:1 one {Un} other {# articles}}", + "fr", + ); !hasCode(issues, CodeSelectorMismatch) { + t.Fatalf("Compare issues = %#v, want decimal offset mismatch", issues) + } +} + +func TestParseUsesICUPatternWhitespace(t *testing.T) { + if _, err := Parse("{\u200ename\u200e,\u200eplural\u200e,\u200eone {One} other {Other}}"); err != nil { + t.Fatalf("Parse rejected Pattern_White_Space separator: %v", err) + } + parsed, err := Parse("{\u00a0name, plural, one {One} other {Other}}") + if err != nil { + t.Fatalf("Parse rejected NBSP as part of an ICU identifier: %v", err) + } + if got := parsed.String(); !strings.HasPrefix(got, "{\u00a0name,") { + t.Fatalf("String() = %q, NBSP was incorrectly consumed as syntax whitespace", got) + } +} + func TestParseValidatesSelectKeywords(t *testing.T) { if _, err := Parse("{g, select, male,female {x} other {y}}"); err == nil { t.Fatal("Parse accepted punctuation in select keyword") diff --git a/internal/translate/protected_test.go b/internal/translate/protected_test.go index 7f535d5..3fe76ac 100644 --- a/internal/translate/protected_test.go +++ b/internal/translate/protected_test.go @@ -22,3 +22,11 @@ func TestValidateTranslationValuePreservesProtectedMarkdown(t *testing.T) { }) } } + +func TestValidateTranslationValueAllowsTargetOnlyICUPluralBranches(t *testing.T) { + source := `{count, plural, one {{name} has one item} other {{name} has # items}}` + target := `{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} many {{name} имеет # предметов} other {{name} имеет # предмета}}` + if err := validateTranslationValue("items", source, target, "ru"); err != nil { + t.Fatalf("valid target-only ICU branches were rejected: %v", err) + } +} diff --git a/internal/validate/protected.go b/internal/validate/protected.go index 2feffd9..7531ccf 100644 --- a/internal/validate/protected.go +++ b/internal/validate/protected.go @@ -4,6 +4,8 @@ import ( "fmt" "regexp" "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/message" ) var ( @@ -16,8 +18,10 @@ var ( // findings with the same stable code and distinct messages. func ProtectedFindings(key, source, target string) []Finding { var findings []Finding - if mismatch := InterpolationMismatch(key, source, target); mismatch != nil { - findings = append(findings, protectedFinding(key, "interpolation variables", mismatch.SourceVars, mismatch.TargetVars)) + if !usesICUValidation(source, target) { + if mismatch := InterpolationMismatch(key, source, target); mismatch != nil { + findings = append(findings, protectedFinding(key, "interpolation variables", mismatch.SourceVars, mismatch.TargetVars)) + } } if sourceTags, targetTags := htmlTagRe.FindAllString(source, -1), htmlTagRe.FindAllString(target, -1); !equalStrings(sourceTags, targetTags) { findings = append(findings, protectedFinding(key, "HTML structure", sourceTags, targetTags)) @@ -34,6 +38,17 @@ func ProtectedFindings(key, source, target string) []Finding { return findings } +func usesICUValidation(source, target string) bool { + if !message.LooksLike(source) || !message.LooksLike(target) { + return false + } + if _, err := message.Parse(source); err != nil { + return false + } + _, err := message.Parse(target) + return err == nil +} + func protectedFinding(key, structure string, expected, actual []string) Finding { return Finding{ Code: CodeProtectedStructureMismatch, diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 38bfb77..418b901 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -141,7 +141,8 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s validationKeys, requiredPluralKeys, optionalPluralKeys = ExpandI18nextV4Source(sourceKeys, sourceLocale, locale) } sourceICUValid := make(map[string]bool, len(validationKeys)) - for key, sourceValue := range validationKeys { + for _, key := range allKeys(validationKeys) { + sourceValue := validationKeys[key] findings := ICUSourceFindings(key, sourceValue, sourceLocale) sourceICUValid[key] = len(findings) == 0 report.Findings = append(report.Findings, findings...) @@ -156,6 +157,7 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s report.Missing = append(report.Missing, key) report.Findings = append(report.Findings, missingFinding(key, requiredPluralKeys)) } + sortFindings(report.Findings) return report } @@ -163,6 +165,7 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s if err != nil { report.Missing = allKeys(validationKeys) report.Errors = append(report.Errors, fmt.Sprintf("parsing target: %v", err)) + sortFindings(report.Findings) return report } @@ -201,8 +204,10 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s report.Findings = append(report.Findings, Finding{Code: CodeSourceIdentical, Severity: SeverityError, Key: key, Message: "target is identical to the source without an exact glossary exemption"}) } - if mismatch := InterpolationMismatch(key, sourceValue, targetValue); mismatch != nil { - report.Mismatches = append(report.Mismatches, *mismatch) + if !usesICUValidation(sourceValue, targetValue) { + if mismatch := InterpolationMismatch(key, sourceValue, targetValue); mismatch != nil { + report.Mismatches = append(report.Mismatches, *mismatch) + } } if sourceICUValid[key] { report.Findings = append(report.Findings, ICUFindings(key, sourceValue, targetValue, locale)...) diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go index 996aa49..c61bb3e 100644 --- a/internal/validate/validate_test.go +++ b/internal/validate/validate_test.go @@ -136,6 +136,16 @@ func TestValidateAllowsLocalePluralBranchesAndExactSelectors(t *testing.T) { } } +func TestValidateUsesBranchAwareICUArgumentsInsteadOfFlatInterpolationCounts(t *testing.T) { + report := validateMessagePair(t, "ru", + `{count, plural, one {{name} has one item} other {{name} has # items}}`, + `{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} many {{name} имеет # предметов} other {{name} имеет # предмета}}`, + ) + if len(report.Mismatches) != 0 || HasFailures([]Report{report}) { + t.Fatalf("valid target-only ICU branches reported as failure: %#v", report) + } +} + func TestValidateReportsMalformedICUSourceWithoutTarget(t *testing.T) { dir := t.TempDir() sourcePath := filepath.Join(dir, "en.json") @@ -151,6 +161,25 @@ func TestValidateReportsMalformedICUSourceWithoutTarget(t *testing.T) { } } +func TestValidateSortsFindingsWhenTargetIsMissing(t *testing.T) { + dir := t.TempDir() + sourcePath := filepath.Join(dir, "en.json") + if err := os.WriteFile(sourcePath, []byte(`{"z":"{n, plural, one {One}}","a":"{n, plural, one {One}}"}`), 0o644); err != nil { + t.Fatal(err) + } + reports, err := Validate(&config.Config{SourceLocale: "en", TargetLocales: []string{"fr"}, SourcePath: sourcePath}) + if err != nil { + t.Fatal(err) + } + if len(reports) != 1 || len(reports[0].Findings) != 4 { + t.Fatalf("reports = %#v, want four findings", reports) + } + got := reports[0].Findings + if got[0].Key != "a" || got[1].Key != "a" || got[2].Key != "z" || got[3].Key != "z" { + t.Fatalf("finding keys = [%q %q %q %q], want deterministic key order", got[0].Key, got[1].Key, got[2].Key, got[3].Key) + } +} + func validateMessagePair(t *testing.T, locale, source, target string) Report { t.Helper() dir := t.TempDir() From 709ba1fb62c29e8dd2a55b1209725c71c3e22d03 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:33:02 -0400 Subject: [PATCH 07/10] fix: cover remaining ICU locale grammar cases Signed-off-by: Tom Main --- internal/locale/locale.go | 9 +++++++-- internal/locale/locale_test.go | 1 + internal/message/message.go | 18 ++++++++++++++++-- internal/message/message_test.go | 11 +++++++++++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/internal/locale/locale.go b/internal/locale/locale.go index 00e9807..0350caf 100644 --- a/internal/locale/locale.go +++ b/internal/locale/locale.go @@ -179,7 +179,7 @@ func needsCurrentCardinalOne(tag language.Tag) bool { base, _ := tag.Base() // These languages postdate x/text's embedded plural tables. switch base.String() { - case "lld", "scn", "vec": + case "bal", "lld", "scn", "vec": return true default: return false @@ -205,5 +205,10 @@ func needsCurrentOrdinalMany(tag language.Tag) bool { func needsCurrentOrdinalOne(tag language.Tag) bool { base, _ := tag.Base() - return base.String() == "kw" + switch base.String() { + case "bal", "kw": + return true + default: + return false + } } diff --git a/internal/locale/locale_test.go b/internal/locale/locale_test.go index 91072cf..816778b 100644 --- a/internal/locale/locale_test.go +++ b/internal/locale/locale_test.go @@ -71,6 +71,7 @@ func TestSupportedLocalesMatchCLDR49CategorySets(t *testing.T) { ordinal []string }{ "ar": {[]string{"zero", "one", "two", "few", "many", "other"}, []string{"other"}}, + "bal": {[]string{"one", "other"}, []string{"one", "other"}}, "bn": {[]string{"one", "other"}, []string{"one", "two", "few", "many", "other"}}, "cs": {[]string{"one", "few", "many", "other"}, []string{"other"}}, "da": {[]string{"one", "other"}, []string{"other"}}, diff --git a/internal/message/message.go b/internal/message/message.go index 7897d58..88dd2be 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -641,8 +641,22 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { if argumentType == ArgumentNumber || argumentType == ArgumentDate || argumentType == ArgumentTime { if p.consume(',') { styleStart := p.position - for p.position < len(p.input) && p.input[p.position] != '}' { - if p.input[p.position] == '{' { + quoted := false + for p.position < len(p.input) { + current := p.input[p.position] + if current == '\'' { + if p.position+1 < len(p.input) && p.input[p.position+1] == '\'' { + p.position += 2 + continue + } + quoted = !quoted + p.position++ + continue + } + if current == '}' && !quoted { + break + } + if current == '{' && !quoted { return nil, p.errorf("unexpected opening brace in %s style", argumentType) } p.position++ diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 1a160eb..1a99510 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -179,6 +179,17 @@ func TestParseUsesICUPatternWhitespace(t *testing.T) { } } +func TestParseAllowsApostropheQuotedBracesInFormatterStyles(t *testing.T) { + input := "{d, date, '{'yyyy'}'}" + parsed, err := Parse(input) + if err != nil { + t.Fatalf("Parse rejected quoted formatter braces: %v", err) + } + if got := parsed.String(); got != "{d, date, '{'yyyy'}'}" { + t.Fatalf("String() = %q, want quoted formatter style preserved", got) + } +} + func TestParseValidatesSelectKeywords(t *testing.T) { if _, err := Parse("{g, select, male,female {x} other {y}}"); err == nil { t.Fatal("Parse accepted punctuation in select keyword") From 569ca2ac55dbe1af04a821b4b1b224280f5b7f7e Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:40:34 -0400 Subject: [PATCH 08/10] fix: tighten ICU syntax classification Signed-off-by: Tom Main --- internal/message/message.go | 21 +++++++++++--------- internal/message/message_test.go | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/internal/message/message.go b/internal/message/message.go index 88dd2be..45f91dd 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -97,7 +97,7 @@ func Parse(input string) (*Message, error) { // the other interpolation syntaxes Internationalizer already supports. func LooksLike(input string) bool { for index := 0; index < len(input); index++ { - if input[index] != '{' || (index > 0 && (input[index-1] == '{' || input[index-1] == '%')) || (index+1 < len(input) && input[index+1] == '{') { + if input[index] != '{' || (index > 0 && input[index-1] == '{') || (index+1 < len(input) && input[index+1] == '{') { continue } cursor := index + 1 @@ -126,6 +126,9 @@ func LooksLike(input string) bool { } cursor += width } + if index > 0 && input[index-1] == '%' && cursor < len(input) && input[cursor] == '}' { + continue + } if cursor >= len(input) || input[cursor] == '}' || input[cursor] == ',' { return true } @@ -661,7 +664,9 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { } p.position++ } - argument.Style = strings.TrimSpace(p.input[styleStart:p.position]) + argument.Style = strings.TrimFunc(p.input[styleStart:p.position], func(character rune) bool { + return unicode.Is(unicode.Pattern_White_Space, character) + }) if argument.Style == "" { return nil, p.errorf("%s style must not be empty", argumentType) } @@ -710,7 +715,7 @@ func (p *parser) parseArgument(pluralContext bool) (*Argument, error) { return nil, p.errorf("duplicate selector %q for argument %q", selector, name) } if argumentType == ArgumentSelect { - if !isIdentifier(selector) { + if !isSelectKeyword(selector) { return nil, p.errorf("invalid select keyword %q", selector) } } else if !isPluralSelector(selector) { @@ -806,16 +811,14 @@ func identifierWidth(input string, position int) int { return width } -func isIdentifier(input string) bool { - if input == "" { +func isSelectKeyword(input string) bool { + if input == "" || !utf8.ValidString(input) { return false } - for position := 0; position < len(input); { - width := identifierWidth(input, position) - if width == 0 { + for _, character := range input { + if unicode.Is(unicode.Pattern_Syntax, character) || unicode.Is(unicode.Pattern_White_Space, character) { return false } - position += width } return true } diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 1a99510..17c21f5 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -190,10 +190,43 @@ func TestParseAllowsApostropheQuotedBracesInFormatterStyles(t *testing.T) { } } +func TestLooksLikeTypedICUArgumentAfterPercentLiteral(t *testing.T) { + source := "%{amount, number}" + if !LooksLike(source) { + t.Fatalf("LooksLike(%q) = false", source) + } + if issues := Compare(source, "%", "fr"); !hasCode(issues, CodeArgumentMismatch) { + t.Fatalf("Compare issues = %#v, want missing typed argument", issues) + } + if LooksLike("%{amount}") { + t.Fatal("LooksLike treated legacy percent interpolation as ICU") + } +} + +func TestComparePreservesNonPatternWhitespaceInFormatterStyles(t *testing.T) { + source := "{d, date,\u00a0yyyy}" + target := "{d, date, yyyy}" + parsed, err := Parse(source) + if err != nil { + t.Fatal(err) + } + if got := parsed.String(); !strings.Contains(got, "\u00a0yyyy") { + t.Fatalf("String() = %q, want NBSP preserved", got) + } + if issues := Compare(source, target, "fr"); !hasCode(issues, CodeArgumentStyleMismatch) { + t.Fatalf("Compare issues = %#v, want formatter style mismatch", issues) + } +} + func TestParseValidatesSelectKeywords(t *testing.T) { if _, err := Parse("{g, select, male,female {x} other {y}}"); err == nil { t.Fatal("Parse accepted punctuation in select keyword") } + for _, keyword := range []string{"male-female", "admin.user"} { + if _, err := Parse("{g, select, " + keyword + " {x} other {y}}"); err == nil { + t.Fatalf("Parse accepted Pattern_Syntax in select keyword %q", keyword) + } + } if _, err := Parse("{g, select, мужчина {x} other {y}}"); err != nil { t.Fatalf("Parse rejected Unicode select keyword: %v", err) } From 171a58401bb7651954e22cdf28849cd277da46a0 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:51:36 -0400 Subject: [PATCH 09/10] fix: validate protected content per ICU branch Signed-off-by: Tom Main --- internal/message/message.go | 61 ++++++++++++++++++++++++++++ internal/message/message_test.go | 34 ++++++++++++++++ internal/translate/protected.go | 2 +- internal/translate/protected_test.go | 4 +- internal/validate/protected.go | 53 ++++++++++++++++++------ internal/validate/strict_test.go | 25 +++++++++++- internal/validate/validate.go | 2 +- 7 files changed, 164 insertions(+), 17 deletions(-) diff --git a/internal/message/message.go b/internal/message/message.go index 45f91dd..95b84eb 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -164,6 +164,67 @@ func Compare(source, target, targetLocale string) []Issue { return issues } +const textTokenArgumentPrefix = "\x00protected:" + +// PreservedTextTokenKinds reports which protected literal token kinds match +// within corresponding ICU branches, including target-only locale branches. +func PreservedTextTokenKinds(source, target, targetLocale string, tokenizers []func(string) []string) ([]bool, error) { + sourceMessage, err := Parse(source) + if err != nil { + return nil, err + } + targetMessage, err := Parse(target) + if err != nil { + return nil, err + } + addTextTokenArguments(sourceMessage, tokenizers) + addTextTokenArguments(targetMessage, tokenizers) + preserved := make([]bool, len(tokenizers)) + for index := range preserved { + preserved[index] = true + } + for _, issue := range compareMessages(sourceMessage, targetMessage, targetLocale, "") { + for index := range tokenizers { + prefix := fmt.Sprintf("%s%08d:", textTokenArgumentPrefix, index) + if strings.HasPrefix(issue.Argument, prefix) { + preserved[index] = false + } + } + } + return preserved, nil +} + +func addTextTokenArguments(message *Message, tokenizers []func(string) []string) { + elementCount := len(message.elements) + var direct strings.Builder + for _, element := range message.elements[:elementCount] { + switch element.kind { + case elementText: + direct.WriteString(element.text) + case elementArgument: + direct.WriteByte('{') + direct.WriteString(element.argument.Name) + direct.WriteByte('}') + case elementPound: + direct.WriteByte('#') + } + } + for kind, tokenize := range tokenizers { + for index, token := range tokenize(direct.String()) { + name := fmt.Sprintf("%s%08d:%08d:%s", textTokenArgumentPrefix, kind, index, token) + message.elements = append(message.elements, element{kind: elementArgument, argument: &Argument{Name: name}}) + } + } + for _, element := range message.elements[:elementCount] { + if element.kind != elementArgument { + continue + } + for _, option := range element.argument.Options { + addTextTokenArguments(option.Message, tokenizers) + } + } +} + func compareMessages(source, target *Message, targetLocale, parentArgument string) []Issue { sourceArguments := immediateArguments(source) targetArguments := immediateArguments(target) diff --git a/internal/message/message_test.go b/internal/message/message_test.go index 17c21f5..a9a28ba 100644 --- a/internal/message/message_test.go +++ b/internal/message/message_test.go @@ -1,6 +1,7 @@ package message import ( + "regexp" "slices" "strings" "testing" @@ -61,6 +62,39 @@ func TestCompareChecksArgumentsAndSelectors(t *testing.T) { } } +func TestPreservedTextTokenKindsCompareCorrespondingPluralBranches(t *testing.T) { + tagPattern := regexp.MustCompile(``) + codePattern := regexp.MustCompile("`[^`]+`") + tokenizers := []func(string) []string{ + func(input string) []string { return tagPattern.FindAllString(input, -1) }, + func(input string) []string { return codePattern.FindAllString(input, -1) }, + } + source := "{n, plural, one {{name} `go test`} other {{name} `go test`}}" + target := "{n, plural, one {{name} `go test`} few {{name} `go test`} many {{name} `go test`} other {{name} `go test`}}" + preserved, err := PreservedTextTokenKinds(source, target, "ru", tokenizers) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(preserved, []bool{true, true}) { + t.Fatalf("PreservedTextTokenKinds() = %v, want both token kinds preserved", preserved) + } + + damaged := "{n, plural, one {{name} `go test`} few {{name} `go test`} many {{name} `go test`} other {{name} `go test`}}" + preserved, err = PreservedTextTokenKinds(source, damaged, "ru", tokenizers) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(preserved, []bool{false, true}) { + t.Fatalf("PreservedTextTokenKinds() = %v, want only tag mismatch", preserved) + } + if _, err := PreservedTextTokenKinds("{broken", target, "ru", tokenizers); err == nil { + t.Fatal("PreservedTextTokenKinds accepted malformed source") + } + if _, err := PreservedTextTokenKinds(source, "{broken", "ru", tokenizers); err == nil { + t.Fatal("PreservedTextTokenKinds accepted malformed target") + } +} + func TestCompareRejectsLocaleInapplicablePluralCategory(t *testing.T) { issues := Compare( "{count, plural, one {One} other {# items}}", diff --git a/internal/translate/protected.go b/internal/translate/protected.go index dc25d6a..b3914bb 100644 --- a/internal/translate/protected.go +++ b/internal/translate/protected.go @@ -14,7 +14,7 @@ func validateTranslationValue(key, source, target, targetLocale string) error { if strings.TrimSpace(source) != "" && strings.TrimSpace(target) == "" { return fmt.Errorf("blank translation for %q", key) } - if findings := validation.ProtectedFindings(key, source, target); len(findings) > 0 { + if findings := validation.ProtectedFindings(key, source, target, targetLocale); len(findings) > 0 { return fmt.Errorf("%s for %q", findings[0].Message, key) } if findings := validation.ICUFindings(key, source, target, targetLocale); len(findings) > 0 { diff --git a/internal/translate/protected_test.go b/internal/translate/protected_test.go index 3fe76ac..9574c72 100644 --- a/internal/translate/protected_test.go +++ b/internal/translate/protected_test.go @@ -24,8 +24,8 @@ func TestValidateTranslationValuePreservesProtectedMarkdown(t *testing.T) { } func TestValidateTranslationValueAllowsTargetOnlyICUPluralBranches(t *testing.T) { - source := `{count, plural, one {{name} has one item} other {{name} has # items}}` - target := `{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} many {{name} имеет # предметов} other {{name} имеет # предмета}}` + source := `{count, plural, one {{name} has one item} other {{name} has # items}}` + target := `{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} many {{name} имеет # предметов} other {{name} имеет # предмета}}` if err := validateTranslationValue("items", source, target, "ru"); err != nil { t.Fatalf("valid target-only ICU branches were rejected: %v", err) } diff --git a/internal/validate/protected.go b/internal/validate/protected.go index 7531ccf..27a3c5c 100644 --- a/internal/validate/protected.go +++ b/internal/validate/protected.go @@ -16,28 +16,57 @@ var ( // ProtectedFindings compares source and target structures that translations // must preserve exactly. Multiple damaged structures may yield multiple // findings with the same stable code and distinct messages. -func ProtectedFindings(key, source, target string) []Finding { +func ProtectedFindings(key, source, target, targetLocale string) []Finding { var findings []Finding - if !usesICUValidation(source, target) { + icu := usesICUValidation(source, target) + if !icu { if mismatch := InterpolationMismatch(key, source, target); mismatch != nil { findings = append(findings, protectedFinding(key, "interpolation variables", mismatch.SourceVars, mismatch.TargetVars)) } } - if sourceTags, targetTags := htmlTagRe.FindAllString(source, -1), htmlTagRe.FindAllString(target, -1); !equalStrings(sourceTags, targetTags) { - findings = append(findings, protectedFinding(key, "HTML structure", sourceTags, targetTags)) - } - if sourceBlocks, targetBlocks := extractFencedCode(source), extractFencedCode(target); !equalStrings(sourceBlocks, targetBlocks) { - findings = append(findings, protectedFinding(key, "fenced code", sourceBlocks, targetBlocks)) - } - if sourceCode, targetCode := inlineCodeRe.FindAllString(source, -1), inlineCodeRe.FindAllString(target, -1); !equalStrings(sourceCode, targetCode) { - findings = append(findings, protectedFinding(key, "inline code", sourceCode, targetCode)) + checks := []struct { + name string + extract func(string) []string + }{ + {"HTML structure", extractHTMLTags}, + {"fenced code", extractFencedCode}, + {"inline code", extractInlineCode}, + {"markdown link destinations", extractLinkDestinations}, + } + tokenizers := make([]func(string) []string, len(checks)) + for index, check := range checks { + tokenizers[index] = check.extract + } + preservedKinds := make([]bool, len(checks)) + if icu { + var err error + preservedKinds, err = message.PreservedTextTokenKinds(source, target, targetLocale, tokenizers) + if err != nil { + icu = false + } } - if sourceLinks, targetLinks := extractLinkDestinations(source), extractLinkDestinations(target); !equalStrings(sourceLinks, targetLinks) { - findings = append(findings, protectedFinding(key, "markdown link destinations", sourceLinks, targetLinks)) + for index, check := range checks { + sourceTokens := check.extract(source) + targetTokens := check.extract(target) + preserved := equalStrings(sourceTokens, targetTokens) + if icu { + preserved = preservedKinds[index] + } + if !preserved { + findings = append(findings, protectedFinding(key, check.name, sourceTokens, targetTokens)) + } } return findings } +func extractHTMLTags(input string) []string { + return htmlTagRe.FindAllString(input, -1) +} + +func extractInlineCode(input string) []string { + return inlineCodeRe.FindAllString(input, -1) +} + func usesICUValidation(source, target string) bool { if !message.LooksLike(source) || !message.LooksLike(target) { return false diff --git a/internal/validate/strict_test.go b/internal/validate/strict_test.go index 1369f3c..6ad429a 100644 --- a/internal/validate/strict_test.go +++ b/internal/validate/strict_test.go @@ -338,7 +338,7 @@ func TestProtectedFindingsUseStableCodeForEveryStructure(t *testing.T) { } for name, values := range tests { t.Run(name, func(t *testing.T) { - findings := ProtectedFindings("key", values[0], values[1]) + findings := ProtectedFindings("key", values[0], values[1], "fr") if len(findings) == 0 || findings[0].Code != CodeProtectedStructureMismatch { t.Fatalf("findings = %#v", findings) } @@ -346,6 +346,29 @@ func TestProtectedFindingsUseStableCodeForEveryStructure(t *testing.T) { } } +func TestProtectedFindingsCompareICUStructuresPerLocaleBranch(t *testing.T) { + tests := map[string]string{ + "html": "Save", + "inline code": "`go test`", + "fenced code": "```go\ngo test\n```\n", + "link": "[Guide](https://example.com/guide)", + } + for name, protected := range tests { + t.Run(name, func(t *testing.T) { + source := "{n, plural, one {" + protected + "} other {" + protected + "}}" + target := "{n, plural, one {" + protected + "} few {" + protected + "} many {" + protected + "} other {" + protected + "}}" + if findings := ProtectedFindings("key", source, target, "ru"); len(findings) != 0 { + t.Fatalf("valid target-only ICU branches produced findings: %#v", findings) + } + + damaged := "{n, plural, one {" + protected + "} few {damaged} many {" + protected + "} other {" + protected + "}}" + if findings := ProtectedFindings("key", source, damaged, "ru"); len(findings) == 0 { + t.Fatal("damaged target-only ICU branch was accepted") + } + }) + } +} + func TestEvaluationCorpusSchemaAndIDsAreStable(t *testing.T) { data, err := os.ReadFile(filepath.Join("..", "..", "test", "evaluation", "v1", "cases.json")) if err != nil { diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 418b901..54714ab 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -213,7 +213,7 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s report.Findings = append(report.Findings, ICUFindings(key, sourceValue, targetValue, locale)...) } if opts.Strict { - report.Findings = append(report.Findings, ProtectedFindings(key, sourceValue, targetValue)...) + report.Findings = append(report.Findings, ProtectedFindings(key, sourceValue, targetValue, locale)...) report.Findings = append(report.Findings, glossaryFindings(key, sourceValue, targetValue, terms)...) } if opts.RequireState { From 168f8a4c2a3c5295a60d3c49cf1bda826c9940fc Mon Sep 17 00:00:00 2001 From: Tom Main Date: Thu, 3 Sep 2026 20:59:16 -0400 Subject: [PATCH 10/10] fix: propagate protected ICU branch context Signed-off-by: Tom Main --- internal/message/message.go | 47 ++++++++++++++++++++++++++------ internal/validate/strict_test.go | 21 ++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/internal/message/message.go b/internal/message/message.go index 95b84eb..5eeeeca 100644 --- a/internal/message/message.go +++ b/internal/message/message.go @@ -177,8 +177,8 @@ func PreservedTextTokenKinds(source, target, targetLocale string, tokenizers []f if err != nil { return nil, err } - addTextTokenArguments(sourceMessage, tokenizers) - addTextTokenArguments(targetMessage, tokenizers) + addTextTokenArguments(sourceMessage, tokenizers, make([]bool, len(tokenizers))) + addTextTokenArguments(targetMessage, tokenizers, make([]bool, len(tokenizers))) preserved := make([]bool, len(tokenizers)) for index := range preserved { preserved[index] = true @@ -194,25 +194,50 @@ func PreservedTextTokenKinds(source, target, targetLocale string, tokenizers []f return preserved, nil } -func addTextTokenArguments(message *Message, tokenizers []func(string) []string) { +func addTextTokenArguments(message *Message, tokenizers []func(string) []string, inheritedProtection []bool) { elementCount := len(message.elements) var direct strings.Builder - for _, element := range message.elements[:elementCount] { + markers := make(map[*Argument]string) + for index, element := range message.elements[:elementCount] { switch element.kind { case elementText: direct.WriteString(element.text) case elementArgument: - direct.WriteByte('{') - direct.WriteString(element.argument.Name) - direct.WriteByte('}') + marker := fmt.Sprintf("\x00icu:%08d:%s\x00", index, element.argument.Name) + markers[element.argument] = marker + direct.WriteString(marker) case elementPound: direct.WriteByte('#') } } + childProtection := make(map[*Argument][]bool) + protectChild := func(argument *Argument, kind int) { + protection := childProtection[argument] + if protection == nil { + protection = make([]bool, len(tokenizers)) + childProtection[argument] = protection + } + protection[kind] = true + } for kind, tokenize := range tokenizers { - for index, token := range tokenize(direct.String()) { + tokens := tokenize(direct.String()) + if inheritedProtection[kind] { + tokens = []string{direct.String()} + for argument := range markers { + protectChild(argument, kind) + } + } + for index, token := range tokens { name := fmt.Sprintf("%s%08d:%08d:%s", textTokenArgumentPrefix, kind, index, token) message.elements = append(message.elements, element{kind: elementArgument, argument: &Argument{Name: name}}) + if inheritedProtection[kind] { + continue + } + for argument, marker := range markers { + if strings.Contains(token, marker) { + protectChild(argument, kind) + } + } } } for _, element := range message.elements[:elementCount] { @@ -220,7 +245,11 @@ func addTextTokenArguments(message *Message, tokenizers []func(string) []string) continue } for _, option := range element.argument.Options { - addTextTokenArguments(option.Message, tokenizers) + protection := childProtection[element.argument] + if protection == nil { + protection = make([]bool, len(tokenizers)) + } + addTextTokenArguments(option.Message, tokenizers, protection) } } } diff --git a/internal/validate/strict_test.go b/internal/validate/strict_test.go index 6ad429a..6a344f1 100644 --- a/internal/validate/strict_test.go +++ b/internal/validate/strict_test.go @@ -369,6 +369,27 @@ func TestProtectedFindingsCompareICUStructuresPerLocaleBranch(t *testing.T) { } } +func TestProtectedFindingsCarryContextIntoNestedICUBranches(t *testing.T) { + argument := "{os, select, win {dir} other {ls}}" + damagedArgument := "{os, select, win {répertoire} other {ls}}" + tests := map[string][2]string{ + "html attribute": {`Run`, `Exécuter`}, + "inline code": {"`" + argument + "`", "`" + damagedArgument + "`"}, + "fenced code": {"```sh\n" + argument + "\n```\n", "```sh\n" + damagedArgument + "\n```\n"}, + "link": {"[Run](" + argument + ")", "[Exécuter](" + damagedArgument + ")"}, + } + for name, values := range tests { + t.Run(name, func(t *testing.T) { + if findings := ProtectedFindings("key", values[0], values[0], "fr"); len(findings) != 0 { + t.Fatalf("unchanged nested ICU protection produced findings: %#v", findings) + } + if findings := ProtectedFindings("key", values[0], values[1], "fr"); len(findings) == 0 { + t.Fatal("damaged nested ICU protected content was accepted") + } + }) + } +} + func TestEvaluationCorpusSchemaAndIDsAreStable(t *testing.T) { data, err := os.ReadFile(filepath.Join("..", "..", "test", "evaluation", "v1", "cases.json")) if err != nil {