diff --git a/README.md b/README.md
index 6f7009d..179dbf7 100644
--- a/README.md
+++ b/README.md
@@ -138,8 +138,9 @@ baseline.
### `validate`
Check all locale files against their source bundles. Default validation checks
-structural coverage (the percentage of required target keys present), reports extra keys
-as warnings, and fails for missing keys or interpolation mismatches.
+structural coverage (the percentage of required target keys present), reports
+extra keys as warnings, and fails for missing keys, interpolation mismatches,
+or invalid ICU MessageFormat structure.
```bash
internationalizer validate # human-readable output
@@ -170,6 +171,9 @@ Human and JSON reports use stable finding codes:
| `protected_structure_mismatch` | Interpolation, HTML, code, or link structure changed |
| `glossary_violation` | No approved target term or variant was found |
| `plural_form_missing` | A configured locale plural form is absent |
+| `icu_message_syntax` | A source or target ICU message is malformed |
+| `icu_argument_mismatch` | ICU argument names, types, or formatter styles differ |
+| `icu_selector_mismatch` | Selectors differ or a plural category is invalid for the target locale |
| `untracked` | No manifest record exists for the target |
| `source_stale` | Source content changed after the recorded translation |
| `policy_stale` | The generated prompt or model settings changed |
@@ -299,6 +303,18 @@ validation:
plural_style: i18next-v4 # generate and validate target-locale plural forms
```
+Locale identifiers must be well-formed BCP 47 tags such as `fr`, `pt-BR`, or
+`sr-Latn-RS`. Canonical-equivalent target locales are rejected as duplicates,
+and locale-specific provider overrides match canonical-equivalent spelling.
+
+ICU MessageFormat values are parsed structurally. Simple arguments, `select`,
+`plural`, `selectordinal`, `number`, `date`, and `time` are supported, including
+nested messages, plural offsets, exact-number selectors, and `#`. Validation
+checks syntax, argument types and formatter styles, plural offsets, select
+branch identity, and target-locale CLDR plural categories. Provider output that
+breaks these invariants is rejected before a locale file or translation-memory
+record is written.
+
With `i18next-v4`, recognized source plural families are expanded during
translation to the target locale's CLDR categories. A target-only category uses
the source family's `_other` value as its translation template. Strict
@@ -392,6 +408,8 @@ internal/
openai.go OpenAI / compatible backend
gemini.go Google Gemini via AI Studio backend
OpenRouter uses openai.go with custom base_url
+ locale/ BCP 47 identity and CLDR plural categories
+ message/ ICU MessageFormat parser and structural comparison
styleguide/ Style guide loader
tm/ JSONL translation memory
translate/ Translation orchestrator
@@ -415,6 +433,8 @@ internal/
[AGPL-3.0](LICENSE)
+See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for dependency notices.
+
## Contributing
See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines. All contributions require DCO sign-off.
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
new file mode 100644
index 0000000..e6e82e0
--- /dev/null
+++ b/THIRD_PARTY_NOTICES.md
@@ -0,0 +1,32 @@
+# Third-party notices
+
+Internationalizer includes software developed by third parties. Their licenses
+apply to those components; the Internationalizer source remains licensed under
+AGPL-3.0.
+
+## golang.org/x/text
+
+Copyright 2009 The Go Authors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+- Neither the name of Google LLC nor the names of its contributors may be used
+ to endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/cmd/internationalizer/translate.go b/cmd/internationalizer/translate.go
index 1ab9fc0..2fa232e 100644
--- a/cmd/internationalizer/translate.go
+++ b/cmd/internationalizer/translate.go
@@ -48,8 +48,12 @@ func newTranslateCmd() *cobra.Command {
selectedLocales = cfg.TargetLocales
}
needsDefaultProvider := false
- for _, locale := range selectedLocales {
- if _, ok := cfg.LLM.LocaleOverrides[locale]; !ok {
+ for _, requested := range selectedLocales {
+ locale, ok := cfg.ConfiguredTargetLocale(requested)
+ if !ok {
+ return fmt.Errorf("locale %q is not in target_locales", requested)
+ }
+ if !cfg.HasLLMOverrideForLocale(locale) {
needsDefaultProvider = true
break
}
@@ -60,8 +64,12 @@ func newTranslateCmd() *cobra.Command {
return err
}
}
- for _, locale := range selectedLocales {
- if _, ok := cfg.LLM.LocaleOverrides[locale]; !ok {
+ for _, requested := range selectedLocales {
+ locale, ok := cfg.ConfiguredTargetLocale(requested)
+ if !ok {
+ return fmt.Errorf("locale %q is not in target_locales", requested)
+ }
+ if !cfg.HasLLMOverrideForLocale(locale) {
continue
}
localeLLM := cfg.LLMForLocale(locale)
diff --git a/docs/localization-core-v2.md b/docs/localization-core-v2.md
new file mode 100644
index 0000000..a300375
--- /dev/null
+++ b/docs/localization-core-v2.md
@@ -0,0 +1,49 @@
+# Localization Core v2 contract
+
+Internationalizer remains a local-first translation compiler for applications
+that own their runtime. This contract does not add a hosted review service,
+framework runtime, translation CDN, or source-rewriting setup command.
+
+## Locale identity
+
+- Configuration accepts well-formed BCP 47 language tags and preserves their
+ spelling in file paths.
+- Equality, provider overrides, validation, and state identity use canonical
+ BCP 47 form. Canonical-equivalent target locales are duplicates.
+- Private-use language tags remain valid. Path separators and malformed tags
+ are rejected before resolving target paths.
+
+## Message semantics
+
+- ICU messages are parsed structurally rather than treated as brace-shaped
+ interpolation strings.
+- Supported arguments are simple interpolation, `select`, `plural`,
+ `selectordinal`, `number`, `date`, and `time`.
+- Selectors may contain nested messages. Plural arguments support exact-number
+ selectors, offsets, and `#`.
+- Every `select`, `plural`, and `selectordinal` requires `other`.
+- A target must retain source argument names, argument kinds, formatter styles,
+ and plural offsets. Select branch identities are semantic and must be
+ preserved. Target plurals may add exact selectors and locale-applicable CLDR
+ categories.
+- Parsing and printing are deterministic so later source-unit hashing can use
+ canonical message structure.
+
+## Review transfer
+
+Manifest v2 will represent generated, needs-review, approved, and manual
+origins explicitly. Only an exact source-and-policy match may inherit approval.
+Fuzzy candidates, changed policy, or changed message structure require review.
+
+## Source units
+
+Structured documents and source adapters will identify a unit by adapter,
+normalized project-relative path, semantic location, and source structure.
+Formatting offsets alone are not stable identity.
+
+## Compatibility boundary
+
+Existing JSON/YAML key workflows and i18next-v4 plural suffixes remain
+supported. Canonical locale comparison may reject configurations that currently
+spell the same locale more than once. ICU structural failures are deterministic
+validation errors and block provider output from being written.
diff --git a/go.mod b/go.mod
index 3fd2061..ba5e1b7 100644
--- a/go.mod
+++ b/go.mod
@@ -1,9 +1,10 @@
module github.com/Tom-R-Main/Internationalizer
-go 1.25
+go 1.25.0
require (
github.com/spf13/cobra v1.10.2
+ golang.org/x/text v0.36.0
gopkg.in/yaml.v3 v3.0.1
)
diff --git a/go.sum b/go.sum
index eaa9638..1053042 100644
--- a/go.sum
+++ b/go.sum
@@ -8,6 +8,8 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/internal/config/config.go b/internal/config/config.go
index d2107e5..4efac7f 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -4,9 +4,10 @@ import (
"fmt"
"os"
"path/filepath"
- "regexp"
+ "sort"
"strings"
+ localeid "github.com/Tom-R-Main/Internationalizer/internal/locale"
"gopkg.in/yaml.v3"
)
@@ -133,7 +134,7 @@ func (c *Config) LLMForLocale(locale string) LLM {
global.LocaleOverrides = nil
applyLLMDefaults(&global)
- override, ok := c.LLM.LocaleOverrides[locale]
+ override, ok := localeOverride(c.LLM.LocaleOverrides, locale)
if !ok {
return global
}
@@ -165,6 +166,29 @@ func (c *Config) LLMForLocale(locale string) LLM {
return effective
}
+// ConfiguredTargetLocale returns the configured spelling of a target locale
+// that is canonically equivalent to requested.
+func (c *Config) ConfiguredTargetLocale(requested string) (string, bool) {
+ canonical, err := localeid.Canonical(requested)
+ if err != nil {
+ return "", false
+ }
+ for _, candidate := range c.TargetLocales {
+ candidateCanonical, candidateErr := localeid.Canonical(candidate)
+ if candidateErr == nil && candidateCanonical == canonical {
+ return candidate, true
+ }
+ }
+ return "", false
+}
+
+// HasLLMOverrideForLocale reports whether a canonical-equivalent locale has
+// explicit provider settings.
+func (c *Config) HasLLMOverrideForLocale(locale string) bool {
+ _, ok := localeOverride(c.LLM.LocaleOverrides, locale)
+ return ok
+}
+
func (c *Config) Validate() error {
if err := c.ValidateProject(); err != nil {
return err
@@ -177,24 +201,38 @@ func (c *Config) ValidateProject() error {
if c.Validation.PluralStyle != "" && c.Validation.PluralStyle != "i18next-v4" {
return fmt.Errorf("unsupported validation.plural_style %q", c.Validation.PluralStyle)
}
+ sourceLocale := c.SourceLocale
+ if sourceLocale == "" {
+ sourceLocale = "en"
+ }
+ if _, err := localeid.Canonical(sourceLocale); err != nil {
+ return fmt.Errorf("invalid source locale %q: %w", c.SourceLocale, err)
+ }
if len(c.TargetLocales) == 0 {
return fmt.Errorf("target_locales must not be empty")
}
localeSeen := make(map[string]struct{}, len(c.TargetLocales))
for _, locale := range c.TargetLocales {
- if !validLocale.MatchString(locale) {
- return fmt.Errorf("invalid target locale %q", locale)
+ canonical, err := localeid.Canonical(locale)
+ if err != nil {
+ return fmt.Errorf("invalid target locale %q: %w", locale, err)
}
- if _, ok := localeSeen[locale]; ok {
+ if _, ok := localeSeen[canonical]; ok {
return fmt.Errorf("duplicate target locale %q", locale)
}
- localeSeen[locale] = struct{}{}
+ localeSeen[canonical] = struct{}{}
}
+ overrideSeen := make(map[string]struct{}, len(c.LLM.LocaleOverrides))
for locale := range c.LLM.LocaleOverrides {
- if !validLocale.MatchString(locale) {
- return fmt.Errorf("invalid llm locale override %q", locale)
+ canonical, err := localeid.Canonical(locale)
+ if err != nil {
+ return fmt.Errorf("invalid llm locale override %q: %w", locale, err)
}
- if _, ok := localeSeen[locale]; !ok {
+ if _, duplicate := overrideSeen[canonical]; duplicate {
+ return fmt.Errorf("duplicate llm locale override %q", locale)
+ }
+ overrideSeen[canonical] = struct{}{}
+ if _, ok := localeSeen[canonical]; !ok {
return fmt.Errorf("llm locale override %q is not in target_locales", locale)
}
}
@@ -276,8 +314,6 @@ func validateAPIKeyEnv(apiKeyEnv string, checked map[string]struct{}) error {
return nil
}
-var validLocale = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`)
-
// EffectiveBundles returns explicit bundles, or a backward-compatible bundle
// derived from source_path and the historical sibling target convention.
func (c *Config) EffectiveBundles() []Bundle {
@@ -298,8 +334,8 @@ func (c *Config) EffectiveBundles() []Bundle {
// TargetPath resolves a bundle target for a validated locale.
func (b Bundle) TargetPath(locale string) (string, error) {
- if !validLocale.MatchString(locale) {
- return "", fmt.Errorf("invalid target locale %q", locale)
+ if _, err := localeid.Canonical(locale); err != nil {
+ return "", fmt.Errorf("invalid target locale %q: %w", locale, err)
}
if !strings.Contains(b.Target, "{locale}") {
return "", fmt.Errorf("bundle %q target must contain {locale}", b.ID)
@@ -307,6 +343,27 @@ func (b Bundle) TargetPath(locale string) (string, error) {
return filepath.Clean(strings.ReplaceAll(b.Target, "{locale}", locale)), nil
}
+func localeOverride(overrides map[string]LLMOverride, requested string) (LLMOverride, bool) {
+ if override, ok := overrides[requested]; ok {
+ return override, true
+ }
+ canonical, err := localeid.Canonical(requested)
+ if err != nil {
+ return LLMOverride{}, false
+ }
+ keys := make([]string, 0, len(overrides))
+ for key := range overrides {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+ for _, key := range keys {
+ if candidate, candidateErr := localeid.Canonical(key); candidateErr == nil && candidate == canonical {
+ return overrides[key], true
+ }
+ }
+ return LLMOverride{}, false
+}
+
// APIKey returns the resolved API key from the environment.
func (c *Config) APIKey() string {
return os.Getenv(c.LLM.APIKeyEnv)
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 039fa66..052e926 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -3,6 +3,7 @@ package config
import (
"os"
"path/filepath"
+ "strings"
"testing"
)
@@ -249,3 +250,82 @@ func TestValidateProjectRejectsLLMOverrideForUnknownLocale(t *testing.T) {
t.Fatal("ValidateProject accepted an LLM override for an unconfigured locale")
}
}
+
+func TestValidateProjectRejectsInvalidBCP47Locales(t *testing.T) {
+ tests := []struct {
+ name string
+ sourceLocale string
+ targetLocales []string
+ want string
+ }{
+ {name: "source", sourceLocale: "en_US", targetLocales: []string{"fr"}, want: "invalid source locale"},
+ {name: "target", sourceLocale: "en", targetLocales: []string{"fr_FR"}, want: "invalid target locale"},
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ cfg := &Config{
+ SourceLocale: test.sourceLocale,
+ TargetLocales: test.targetLocales,
+ SourcePath: filepath.Join("locales", "en.json"),
+ }
+ err := cfg.ValidateProject()
+ if err == nil || !strings.Contains(err.Error(), test.want) {
+ t.Fatalf("ValidateProject error = %v, want error containing %q", err, test.want)
+ }
+ })
+ }
+}
+
+func TestValidateProjectRejectsCanonicalDuplicateLocales(t *testing.T) {
+ cfg := &Config{
+ SourceLocale: "en",
+ TargetLocales: []string{"en-us", "en-US"},
+ SourcePath: filepath.Join("locales", "en.json"),
+ }
+ err := cfg.ValidateProject()
+ if err == nil || !strings.Contains(err.Error(), "duplicate target locale") {
+ t.Fatalf("ValidateProject error = %v, want canonical duplicate error", err)
+ }
+}
+
+func TestValidateProjectAcceptsBCP47ScriptAndRegion(t *testing.T) {
+ cfg := &Config{
+ SourceLocale: "en-US",
+ TargetLocales: []string{"sr-Latn-RS"},
+ SourcePath: filepath.Join("locales", "en.json"),
+ }
+ if err := cfg.ValidateProject(); err != nil {
+ t.Fatalf("ValidateProject rejected valid BCP 47 locales: %v", err)
+ }
+}
+
+func TestValidateProjectMatchesCanonicalLLMOverrideLocale(t *testing.T) {
+ cfg := &Config{
+ SourceLocale: "en",
+ TargetLocales: []string{"pt-br"},
+ SourcePath: filepath.Join("locales", "en.json"),
+ LLM: LLM{LocaleOverrides: map[string]LLMOverride{
+ "pt-BR": {Provider: "openrouter", Model: "example/model"},
+ }},
+ }
+ if err := cfg.ValidateProject(); err != nil {
+ t.Fatalf("ValidateProject rejected canonical-equivalent override: %v", err)
+ }
+ if got := cfg.LLMForLocale("pt-br"); got.Provider != "openrouter" || got.Model != "example/model" {
+ t.Fatalf("LLMForLocale = %#v, want canonical-equivalent override", got)
+ }
+ if !cfg.HasLLMOverrideForLocale("pt-br") {
+ t.Fatal("HasLLMOverrideForLocale missed canonical-equivalent override")
+ }
+ if got, ok := cfg.ConfiguredTargetLocale("pt-BR"); !ok || got != "pt-br" {
+ t.Fatalf("ConfiguredTargetLocale = %q, %v; want %q, true", got, ok, "pt-br")
+ }
+}
+
+func TestBundleTargetPathStillRejectsUnsafeLocale(t *testing.T) {
+ bundle := Bundle{ID: "app", Target: filepath.Join("locales", "{locale}.json")}
+ if _, err := bundle.TargetPath("../fr"); err == nil {
+ t.Fatal("TargetPath accepted path traversal as a locale")
+ }
+}
diff --git a/internal/formats/fuzz_benchmark_test.go b/internal/formats/fuzz_benchmark_test.go
index 829f5c6..4f44378 100644
--- a/internal/formats/fuzz_benchmark_test.go
+++ b/internal/formats/fuzz_benchmark_test.go
@@ -13,6 +13,7 @@ func FuzzJSONParseSerializeRoundTrip(f *testing.F) {
[]byte(`{"items":["One","Two"],"enabled":true,"count":2,"empty":null}`),
[]byte(`{"numeric":{"0":"Zero"}}`),
[]byte(`{"nested":{"welcome":"Hello, {name}"}}`),
+ []byte(`"Save"`),
[]byte(`not JSON`),
} {
f.Add(seed)
diff --git a/internal/formats/json.go b/internal/formats/json.go
index 1049f7f..9ab2b66 100644
--- a/internal/formats/json.go
+++ b/internal/formats/json.go
@@ -103,6 +103,12 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt
return nil, fmt.Errorf("json parse original: %w", err)
}
replaced := make(map[string]struct{}, len(entries))
+ if _, rootString := raw.(string); rootString {
+ if replacement, ok := entries[""]; ok {
+ raw = replacement
+ replaced[""] = struct{}{}
+ }
+ }
replaceLeaves("", raw, entries, replaced)
for key, value := range entries {
if _, ok := replaced[key]; ok {
diff --git a/internal/formats/json_test.go b/internal/formats/json_test.go
index 6ff0c20..557dd42 100644
--- a/internal/formats/json_test.go
+++ b/internal/formats/json_test.go
@@ -68,6 +68,22 @@ func TestJSONParseExcludesNonStringLeaves(t *testing.T) {
}
}
+func TestJSONRoundTripsRootString(t *testing.T) {
+ f := &JSONFormat{}
+ entries, err := f.Parse([]byte(`"Save"`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ entries[""] = "Enregistrer"
+ output, err := f.Serialize(entries, []byte(`"Save"`))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(output) != `"Enregistrer"` {
+ t.Fatalf("Serialize() = %s, want translated root string", output)
+ }
+}
+
func TestJSONSerializePreservesOrder(t *testing.T) {
original := `{
"b": "B",
diff --git a/internal/locale/locale.go b/internal/locale/locale.go
new file mode 100644
index 0000000..0350caf
--- /dev/null
+++ b/internal/locale/locale.go
@@ -0,0 +1,214 @@
+// Package locale owns canonical locale identity and CLDR plural metadata.
+package locale
+
+import (
+ "fmt"
+ "slices"
+ "strings"
+ "sync"
+
+ "golang.org/x/text/feature/plural"
+ "golang.org/x/text/language"
+)
+
+// PluralType selects cardinal or ordinal CLDR plural rules.
+type PluralType string
+
+const (
+ Cardinal PluralType = "cardinal"
+ Ordinal PluralType = "ordinal"
+)
+
+var categoryCache sync.Map
+
+// Canonical validates a BCP 47 language tag and returns its canonical form.
+func Canonical(input string) (string, error) {
+ tag, err := Parse(input)
+ if err != nil {
+ return "", err
+ }
+ return tag.String(), nil
+}
+
+// Parse validates a BCP 47 language tag. Underscore aliases are deliberately
+// rejected so configuration and persisted state have one unambiguous syntax.
+func Parse(input string) (language.Tag, error) {
+ if input == "" {
+ return language.Tag{}, fmt.Errorf("locale must not be empty")
+ }
+ if strings.ContainsRune(input, '_') {
+ return language.Tag{}, fmt.Errorf("locale must use BCP 47 hyphen separators")
+ }
+ tag, err := language.Parse(input)
+ if err != nil {
+ return language.Tag{}, fmt.Errorf("invalid BCP 47 language tag: %w", err)
+ }
+ return tag, nil
+}
+
+// CardinalCategories returns the CLDR cardinal categories used by a locale.
+func CardinalCategories(input string) ([]string, error) {
+ return categories(input, Cardinal)
+}
+
+// OrdinalCategories returns the CLDR ordinal categories used by a locale.
+func OrdinalCategories(input string) ([]string, error) {
+ return categories(input, Ordinal)
+}
+
+func categories(input string, pluralType PluralType) ([]string, error) {
+ tag, err := Parse(input)
+ if err != nil {
+ return nil, err
+ }
+ key := string(pluralType) + ":" + tag.String()
+ if cached, ok := categoryCache.Load(key); ok {
+ return slices.Clone(cached.([]string)), nil
+ }
+
+ rules := plural.Cardinal
+ if pluralType == Ordinal {
+ rules = plural.Ordinal
+ }
+ seen := make(map[plural.Form]struct{}, 6)
+ record := func(form plural.Form) { seen[form] = struct{}{} }
+
+ // x/text's generated CLDR evaluator differentiates integer rules through
+ // mod 100 plus a few documented large-number exceptions. Cover both the
+ // complete modulus and those exception ranges.
+ for integer := 0; integer <= 2000; integer++ {
+ record(rules.MatchPlural(tag, integer, 0, 0, 0, 0))
+ }
+ for _, integer := range []int{10_000, 100_000, 1_000_000, 2_000_000, 10_000_000} {
+ record(rules.MatchPlural(tag, integer, 0, 0, 0, 0))
+ }
+
+ // Decimal rules can distinguish the number of visible digits and fraction
+ // operands. Values through 199 cover both sides of x/text's bounded rule
+ // representation and all modulo-100 classes.
+ for visibleDigits := 1; visibleDigits <= 4; visibleDigits++ {
+ maxFraction := 1
+ for range visibleDigits {
+ maxFraction *= 10
+ }
+ maxFraction--
+ if maxFraction > 199 {
+ maxFraction = 199
+ }
+ for integer := 0; integer <= 200; integer++ {
+ for fraction := 0; fraction <= maxFraction; fraction++ {
+ record(rules.MatchPlural(tag, integer, visibleDigits, visibleDigits, fraction, fraction))
+ }
+ }
+ }
+
+ // x/text/feature/plural still embeds CLDR 32 rules. Overlay category-set
+ // changes needed by the current CLDR 49 contract; the public evaluator also
+ // cannot represent the compact-exponent operand used by "many" rules.
+ if pluralType == Cardinal && hasCompactExponentMany(tag) {
+ seen[plural.Many] = struct{}{}
+ }
+ if pluralType == Cardinal && hasCurrentCardinalTwo(tag) {
+ seen[plural.Two] = struct{}{}
+ }
+ if pluralType == Cardinal && needsCurrentCardinalOne(tag) {
+ seen[plural.One] = struct{}{}
+ }
+ if pluralType == Cardinal && needsCurrentCardinalZeroFewMany(tag) {
+ seen[plural.Zero] = struct{}{}
+ seen[plural.Few] = struct{}{}
+ seen[plural.Many] = struct{}{}
+ }
+ if pluralType == Ordinal && needsCurrentOrdinalMany(tag) {
+ seen[plural.Many] = struct{}{}
+ }
+ if pluralType == Ordinal && needsCurrentOrdinalOne(tag) {
+ seen[plural.One] = struct{}{}
+ }
+ if pluralType == Cardinal && hasRetiredCardinalMany(tag) {
+ delete(seen, plural.Many)
+ }
+
+ ordered := make([]string, 0, len(seen))
+ for _, candidate := range []struct {
+ form plural.Form
+ name string
+ }{
+ {plural.Zero, "zero"},
+ {plural.One, "one"},
+ {plural.Two, "two"},
+ {plural.Few, "few"},
+ {plural.Many, "many"},
+ {plural.Other, "other"},
+ } {
+ if _, ok := seen[candidate.form]; ok {
+ ordered = append(ordered, candidate.name)
+ }
+ }
+ if !slices.Contains(ordered, "other") {
+ ordered = append(ordered, "other")
+ }
+ categoryCache.Store(key, slices.Clone(ordered))
+ return ordered, nil
+}
+
+func hasCompactExponentMany(tag language.Tag) bool {
+ base, _ := tag.Base()
+ switch base.String() {
+ case "ca", "es", "fr", "gl", "it", "lld", "pt", "scn", "vec":
+ return true
+ default:
+ return false
+ }
+}
+
+func hasRetiredCardinalMany(tag language.Tag) bool {
+ base, _ := tag.Base()
+ // Modern CLDR removed Hebrew's former 20/30/... "many" category. Keep the
+ // current category set even though x/text still evaluates the older rule.
+ return base.String() == "he"
+}
+
+func hasCurrentCardinalTwo(tag language.Tag) bool {
+ base, _ := tag.Base()
+ // CLDR 42 split Maltese cardinal "other" to add a distinct "two" form.
+ return base.String() == "mt"
+}
+
+func needsCurrentCardinalOne(tag language.Tag) bool {
+ base, _ := tag.Base()
+ // These languages postdate x/text's embedded plural tables.
+ switch base.String() {
+ case "bal", "lld", "scn", "vec":
+ return true
+ default:
+ return false
+ }
+}
+
+func needsCurrentCardinalZeroFewMany(tag language.Tag) bool {
+ base, _ := tag.Base()
+ // Modern Cornish distinguishes all six cardinal categories.
+ return base.String() == "kw"
+}
+
+func needsCurrentOrdinalMany(tag language.Tag) bool {
+ base, _ := tag.Base()
+ // Sicilian and Venetian ordinal rules postdate x/text's embedded tables.
+ switch base.String() {
+ case "kw", "lld", "scn", "vec":
+ return true
+ default:
+ return false
+ }
+}
+
+func needsCurrentOrdinalOne(tag language.Tag) bool {
+ base, _ := tag.Base()
+ switch base.String() {
+ case "bal", "kw":
+ return true
+ default:
+ return false
+ }
+}
diff --git a/internal/locale/locale_test.go b/internal/locale/locale_test.go
new file mode 100644
index 0000000..816778b
--- /dev/null
+++ b/internal/locale/locale_test.go
@@ -0,0 +1,127 @@
+package locale
+
+import (
+ "slices"
+ "testing"
+)
+
+func TestCanonical(t *testing.T) {
+ tests := map[string]string{
+ "EN-us": "en-US",
+ "sr-latn-rs": "sr-Latn-RS",
+ "zh-hant-tw": "zh-Hant-TW",
+ }
+ for input, want := range tests {
+ t.Run(input, func(t *testing.T) {
+ got, err := Canonical(input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != want {
+ t.Fatalf("Canonical(%q) = %q, want %q", input, got, want)
+ }
+ })
+ }
+}
+
+func TestCanonicalRejectsMalformedTags(t *testing.T) {
+ for _, input := range []string{"", "en_US", "../fr", "en--US"} {
+ t.Run(input, func(t *testing.T) {
+ if _, err := Canonical(input); err == nil {
+ t.Fatalf("Canonical(%q) succeeded", input)
+ }
+ })
+ }
+}
+
+func TestCardinalCategoriesUseCLDRRules(t *testing.T) {
+ tests := map[string][]string{
+ "ar": {"zero", "one", "two", "few", "many", "other"},
+ "ja": {"other"},
+ "ru": {"one", "few", "many", "other"},
+ "te": {"one", "other"},
+ }
+ for tag, want := range tests {
+ t.Run(tag, func(t *testing.T) {
+ got, err := CardinalCategories(tag)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(got, want) {
+ t.Fatalf("CardinalCategories(%q) = %v, want %v", tag, got, want)
+ }
+ })
+ }
+}
+
+func TestOrdinalCategoriesUseCLDRRules(t *testing.T) {
+ got, err := OrdinalCategories("en-US")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := []string{"one", "two", "few", "other"}
+ if !slices.Equal(got, want) {
+ t.Fatalf("OrdinalCategories(en-US) = %v, want %v", got, want)
+ }
+}
+
+func TestSupportedLocalesMatchCLDR49CategorySets(t *testing.T) {
+ tests := map[string]struct {
+ cardinal []string
+ ordinal []string
+ }{
+ "ar": {[]string{"zero", "one", "two", "few", "many", "other"}, []string{"other"}},
+ "bal": {[]string{"one", "other"}, []string{"one", "other"}},
+ "bn": {[]string{"one", "other"}, []string{"one", "two", "few", "many", "other"}},
+ "cs": {[]string{"one", "few", "many", "other"}, []string{"other"}},
+ "da": {[]string{"one", "other"}, []string{"other"}},
+ "de": {[]string{"one", "other"}, []string{"other"}},
+ "el": {[]string{"one", "other"}, []string{"other"}},
+ "es": {[]string{"one", "many", "other"}, []string{"other"}},
+ "fi": {[]string{"one", "other"}, []string{"other"}},
+ "fr": {[]string{"one", "many", "other"}, []string{"one", "other"}},
+ "he": {[]string{"one", "two", "other"}, []string{"other"}},
+ "hi": {[]string{"one", "other"}, []string{"one", "two", "few", "many", "other"}},
+ "id": {[]string{"other"}, []string{"other"}},
+ "it": {[]string{"one", "many", "other"}, []string{"many", "other"}},
+ "ja": {[]string{"other"}, []string{"other"}},
+ "ko": {[]string{"other"}, []string{"other"}},
+ "kw": {[]string{"zero", "one", "two", "few", "many", "other"}, []string{"one", "many", "other"}},
+ "lld": {[]string{"one", "many", "other"}, []string{"many", "other"}},
+ "ms": {[]string{"other"}, []string{"one", "other"}},
+ "mt": {[]string{"one", "two", "few", "many", "other"}, []string{"other"}},
+ "nl": {[]string{"one", "other"}, []string{"other"}},
+ "pa": {[]string{"one", "other"}, []string{"other"}},
+ "pl": {[]string{"one", "few", "many", "other"}, []string{"other"}},
+ "pt-BR": {[]string{"one", "many", "other"}, []string{"other"}},
+ "ro": {[]string{"one", "few", "other"}, []string{"one", "other"}},
+ "ru": {[]string{"one", "few", "many", "other"}, []string{"other"}},
+ "scn": {[]string{"one", "many", "other"}, []string{"many", "other"}},
+ "sv": {[]string{"one", "other"}, []string{"one", "other"}},
+ "te": {[]string{"one", "other"}, []string{"other"}},
+ "th": {[]string{"other"}, []string{"other"}},
+ "tr": {[]string{"one", "other"}, []string{"other"}},
+ "uk": {[]string{"one", "few", "many", "other"}, []string{"few", "other"}},
+ "vec": {[]string{"one", "many", "other"}, []string{"many", "other"}},
+ "vi": {[]string{"other"}, []string{"one", "other"}},
+ "yue": {[]string{"other"}, []string{"other"}},
+ "zh-CN": {[]string{"other"}, []string{"other"}},
+ "zh-TW": {[]string{"other"}, []string{"other"}},
+ }
+
+ for tag, want := range tests {
+ t.Run(tag, func(t *testing.T) {
+ cardinal, err := CardinalCategories(tag)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ordinal, err := OrdinalCategories(tag)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(cardinal, want.cardinal) || !slices.Equal(ordinal, want.ordinal) {
+ t.Fatalf("%s categories: cardinal=%v ordinal=%v, want cardinal=%v ordinal=%v", tag, cardinal, ordinal, want.cardinal, want.ordinal)
+ }
+ })
+ }
+}
diff --git a/internal/message/message.go b/internal/message/message.go
new file mode 100644
index 0000000..5eeeeca
--- /dev/null
+++ b/internal/message/message.go
@@ -0,0 +1,1033 @@
+// Package message parses and compares ICU MessageFormat messages.
+package message
+
+import (
+ "fmt"
+ "sort"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+
+ localeid "github.com/Tom-R-Main/Internationalizer/internal/locale"
+)
+
+// Code is a stable machine-readable message validation outcome.
+type Code string
+
+const (
+ CodeSyntax Code = "syntax"
+ CodeArgumentMismatch Code = "argument_mismatch"
+ CodeArgumentTypeMismatch Code = "argument_type_mismatch"
+ CodeArgumentStyleMismatch Code = "argument_style_mismatch"
+ CodeSelectorMismatch Code = "selector_mismatch"
+ CodeInvalidPluralCategory Code = "invalid_plural_category"
+)
+
+const maxNestingDepth = 100
+
+// Issue describes one structural incompatibility between source and target.
+type Issue struct {
+ Code Code
+ Argument string
+ Message string
+}
+
+// ArgumentType identifies the ICU operation performed by an argument.
+type ArgumentType string
+
+const (
+ ArgumentSimple ArgumentType = ""
+ ArgumentSelect ArgumentType = "select"
+ ArgumentPlural ArgumentType = "plural"
+ ArgumentSelectOrdinal ArgumentType = "selectordinal"
+ ArgumentNumber ArgumentType = "number"
+ ArgumentDate ArgumentType = "date"
+ ArgumentTime ArgumentType = "time"
+)
+
+type elementKind uint8
+
+const (
+ elementText elementKind = iota
+ elementArgument
+ elementPound
+)
+
+// Message is a parsed ICU message. Its String method emits a deterministic
+// representation suitable for structural hashing.
+type Message struct {
+ elements []element
+}
+
+type element struct {
+ kind elementKind
+ text string
+ argument *Argument
+}
+
+// Argument is one parsed ICU argument.
+type Argument struct {
+ Name string
+ Type ArgumentType
+ Style string
+ Offset string
+ Options []Option
+}
+
+// Option is one select or plural branch.
+type Option struct {
+ Selector string
+ Message *Message
+}
+
+// Parse parses an ICU MessageFormat message.
+func Parse(input string) (*Message, error) {
+ parser := parser{input: input}
+ message, err := parser.parseMessage(false, false)
+ if err != nil {
+ return nil, err
+ }
+ if parser.position != len(input) {
+ return nil, parser.errorf("unexpected trailing input")
+ }
+ return message, nil
+}
+
+// LooksLike reports whether input contains an ICU-shaped argument. It excludes
+// the other interpolation syntaxes Internationalizer already supports.
+func LooksLike(input string) bool {
+ for index := 0; index < len(input); index++ {
+ if input[index] != '{' || (index > 0 && input[index-1] == '{') || (index+1 < len(input) && input[index+1] == '{') {
+ continue
+ }
+ cursor := index + 1
+ for cursor < len(input) {
+ width := spaceWidth(input, cursor)
+ if width == 0 {
+ break
+ }
+ cursor += width
+ }
+ start := cursor
+ for cursor < len(input) {
+ width := identifierWidth(input, cursor)
+ if width == 0 {
+ break
+ }
+ cursor += width
+ }
+ if cursor == start {
+ continue
+ }
+ for cursor < len(input) {
+ width := spaceWidth(input, cursor)
+ if width == 0 {
+ break
+ }
+ cursor += width
+ }
+ if index > 0 && input[index-1] == '%' && cursor < len(input) && input[cursor] == '}' {
+ continue
+ }
+ if cursor >= len(input) || input[cursor] == '}' || input[cursor] == ',' {
+ return true
+ }
+ }
+ return false
+}
+
+// Compare reports structural incompatibilities between ICU-shaped source and
+// target messages. Plain text and non-ICU interpolation are ignored.
+func Compare(source, target, targetLocale string) []Issue {
+ if !LooksLike(source) && !LooksLike(target) {
+ return nil
+ }
+ sourceMessage, err := Parse(source)
+ if err != nil {
+ return []Issue{{Code: CodeSyntax, Message: fmt.Sprintf("source ICU message: %v", err)}}
+ }
+ targetMessage, err := Parse(target)
+ if err != nil {
+ return []Issue{{Code: CodeSyntax, Message: fmt.Sprintf("target ICU message: %v", err)}}
+ }
+
+ issues := compareMessages(sourceMessage, targetMessage, targetLocale, "")
+ sort.Slice(issues, func(left, right int) bool {
+ if issues[left].Argument != issues[right].Argument {
+ return issues[left].Argument < issues[right].Argument
+ }
+ if issues[left].Code != issues[right].Code {
+ return issues[left].Code < issues[right].Code
+ }
+ return issues[left].Message < issues[right].Message
+ })
+ return issues
+}
+
+const textTokenArgumentPrefix = "\x00protected:"
+
+// PreservedTextTokenKinds reports which protected literal token kinds match
+// within corresponding ICU branches, including target-only locale branches.
+func PreservedTextTokenKinds(source, target, targetLocale string, tokenizers []func(string) []string) ([]bool, error) {
+ sourceMessage, err := Parse(source)
+ if err != nil {
+ return nil, err
+ }
+ targetMessage, err := Parse(target)
+ if err != nil {
+ return nil, err
+ }
+ addTextTokenArguments(sourceMessage, tokenizers, make([]bool, len(tokenizers)))
+ addTextTokenArguments(targetMessage, tokenizers, make([]bool, len(tokenizers)))
+ preserved := make([]bool, len(tokenizers))
+ for index := range preserved {
+ preserved[index] = true
+ }
+ for _, issue := range compareMessages(sourceMessage, targetMessage, targetLocale, "") {
+ for index := range tokenizers {
+ prefix := fmt.Sprintf("%s%08d:", textTokenArgumentPrefix, index)
+ if strings.HasPrefix(issue.Argument, prefix) {
+ preserved[index] = false
+ }
+ }
+ }
+ return preserved, nil
+}
+
+func addTextTokenArguments(message *Message, tokenizers []func(string) []string, inheritedProtection []bool) {
+ elementCount := len(message.elements)
+ var direct strings.Builder
+ markers := make(map[*Argument]string)
+ for index, element := range message.elements[:elementCount] {
+ switch element.kind {
+ case elementText:
+ direct.WriteString(element.text)
+ case elementArgument:
+ marker := fmt.Sprintf("\x00icu:%08d:%s\x00", index, element.argument.Name)
+ markers[element.argument] = marker
+ direct.WriteString(marker)
+ case elementPound:
+ direct.WriteByte('#')
+ }
+ }
+ childProtection := make(map[*Argument][]bool)
+ protectChild := func(argument *Argument, kind int) {
+ protection := childProtection[argument]
+ if protection == nil {
+ protection = make([]bool, len(tokenizers))
+ childProtection[argument] = protection
+ }
+ protection[kind] = true
+ }
+ for kind, tokenize := range tokenizers {
+ tokens := tokenize(direct.String())
+ if inheritedProtection[kind] {
+ tokens = []string{direct.String()}
+ for argument := range markers {
+ protectChild(argument, kind)
+ }
+ }
+ for index, token := range tokens {
+ name := fmt.Sprintf("%s%08d:%08d:%s", textTokenArgumentPrefix, kind, index, token)
+ message.elements = append(message.elements, element{kind: elementArgument, argument: &Argument{Name: name}})
+ if inheritedProtection[kind] {
+ continue
+ }
+ for argument, marker := range markers {
+ if strings.Contains(token, marker) {
+ protectChild(argument, kind)
+ }
+ }
+ }
+ }
+ for _, element := range message.elements[:elementCount] {
+ if element.kind != elementArgument {
+ continue
+ }
+ for _, option := range element.argument.Options {
+ protection := childProtection[element.argument]
+ if protection == nil {
+ protection = make([]bool, len(tokenizers))
+ }
+ addTextTokenArguments(option.Message, tokenizers, protection)
+ }
+ }
+}
+
+func compareMessages(source, target *Message, targetLocale, parentArgument string) []Issue {
+ sourceArguments := immediateArguments(source)
+ targetArguments := immediateArguments(target)
+ var issues []Issue
+ if sourcePounds, targetPounds := poundCount(source), poundCount(target); sourcePounds != targetPounds {
+ issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: parentArgument, Message: fmt.Sprintf("ICU plural branch contains %d pound placeholders in the source and %d in the target", sourcePounds, targetPounds)})
+ }
+ for name, sourceOccurrences := range sourceArguments {
+ targetOccurrences, ok := targetArguments[name]
+ if !ok {
+ issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q is missing from the corresponding target branch", name)})
+ continue
+ }
+ if len(sourceOccurrences) != len(targetOccurrences) {
+ issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q occurs %d times in the source branch and %d times in the target branch", name, len(sourceOccurrences), len(targetOccurrences))})
+ }
+ if len(sourceOccurrences) == len(targetOccurrences) && hasPerfectArgumentMatching(sourceOccurrences, targetOccurrences, targetLocale) {
+ continue
+ }
+ sourceOccurrences = sortedArguments(sourceOccurrences)
+ targetOccurrences = sortedArguments(targetOccurrences)
+ occurrences := min(len(sourceOccurrences), len(targetOccurrences))
+ for index := range occurrences {
+ issues = append(issues, compareArguments(sourceOccurrences[index], targetOccurrences[index], targetLocale)...)
+ }
+ }
+ for name := range targetArguments {
+ if _, ok := sourceArguments[name]; !ok {
+ issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q is absent from the corresponding source branch", name)})
+ }
+ }
+ return issues
+}
+
+func hasPerfectArgumentMatching(source, target []*Argument, targetLocale string) bool {
+ matchedSource := make([]int, len(target))
+ for index := range matchedSource {
+ matchedSource[index] = -1
+ }
+ var match func(int, []bool) bool
+ match = func(sourceIndex int, visited []bool) bool {
+ for targetIndex := range target {
+ if visited[targetIndex] || len(compareArguments(source[sourceIndex], target[targetIndex], targetLocale)) != 0 {
+ continue
+ }
+ visited[targetIndex] = true
+ if matchedSource[targetIndex] == -1 || match(matchedSource[targetIndex], visited) {
+ matchedSource[targetIndex] = sourceIndex
+ return true
+ }
+ }
+ return false
+ }
+ for sourceIndex := range source {
+ if !match(sourceIndex, make([]bool, len(target))) {
+ return false
+ }
+ }
+ return true
+}
+
+func sortedArguments(arguments []*Argument) []*Argument {
+ result := append([]*Argument(nil), arguments...)
+ sort.SliceStable(result, func(left, right int) bool {
+ return argumentSortKey(result[left]) < argumentSortKey(result[right])
+ })
+ return result
+}
+
+func argumentSortKey(argument *Argument) string {
+ options := make([]string, 0, len(argument.Options))
+ for _, option := range argument.Options {
+ options = append(options, option.Selector+"{"+messageSortKey(option.Message)+"}")
+ }
+ sort.Strings(options)
+ return fmt.Sprintf("%s\x00%s\x00%s\x00%s", argument.Type, argument.Style, argument.Offset, strings.Join(options, "\x00"))
+}
+
+func messageSortKey(message *Message) string {
+ parts := make([]string, 0, len(message.elements))
+ for _, element := range message.elements {
+ switch element.kind {
+ case elementArgument:
+ parts = append(parts, element.argument.Name+"{"+argumentSortKey(element.argument)+"}")
+ case elementPound:
+ parts = append(parts, "#")
+ }
+ }
+ sort.Strings(parts)
+ return strings.Join(parts, "\x00")
+}
+
+func poundCount(message *Message) int {
+ count := 0
+ for _, element := range message.elements {
+ if element.kind == elementPound {
+ count++
+ }
+ }
+ return count
+}
+
+func compareArguments(source, target *Argument, targetLocale string) []Issue {
+ name := source.Name
+ if source.Type != target.Type {
+ return []Issue{{Code: CodeArgumentTypeMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q types differ: source %q, target %q", name, displayType(source.Type), displayType(target.Type))}}
+ }
+ var issues []Issue
+ switch source.Type {
+ case ArgumentNumber, ArgumentDate, ArgumentTime:
+ if source.Style != target.Style {
+ issues = append(issues, Issue{Code: CodeArgumentStyleMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q formatter styles differ", name)})
+ }
+ case ArgumentSelect:
+ sourceOptions := optionsBySelector(source)
+ targetOptions := optionsBySelector(target)
+ if !sameOptionSet(sourceOptions, targetOptions) {
+ issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: name, Message: fmt.Sprintf("ICU select argument %q selectors differ", name)})
+ }
+ for selector, sourceBranch := range sourceOptions {
+ if targetBranch, ok := targetOptions[selector]; ok {
+ issues = append(issues, compareMessages(sourceBranch, targetBranch, targetLocale, name)...)
+ }
+ }
+ case ArgumentPlural, ArgumentSelectOrdinal:
+ if source.Offset != target.Offset {
+ issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q plural offsets differ", name)})
+ }
+ issues = append(issues, comparePluralSelectors(source, target, targetLocale)...)
+ sourceOptions := optionsBySelector(source)
+ targetOptions := optionsBySelector(target)
+ for selector, sourceBranch := range sourceOptions {
+ if targetBranch, ok := targetOptions[selector]; ok {
+ issues = append(issues, compareMessages(sourceBranch, targetBranch, targetLocale, name)...)
+ }
+ }
+ sourceOther := sourceOptions["other"]
+ for selector, targetBranch := range targetOptions {
+ if _, sourceHasSelector := sourceOptions[selector]; sourceHasSelector {
+ continue
+ }
+ if isExactSelector(selector) {
+ issues = append(issues, compareTargetArgumentSubset(sourceOther, targetBranch, targetLocale)...)
+ } else {
+ issues = append(issues, compareMessages(sourceOther, targetBranch, targetLocale, name)...)
+ }
+ }
+ }
+ return issues
+}
+
+func compareTargetArgumentSubset(source, target *Message, targetLocale string) []Issue {
+ sourceArguments := immediateArguments(source)
+ targetArguments := immediateArguments(target)
+ var issues []Issue
+ for name, targetOccurrences := range targetArguments {
+ sourceOccurrences, ok := sourceArguments[name]
+ if !ok {
+ issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU exact-selector branch introduces argument %q that is absent from the source fallback branch", name)})
+ continue
+ }
+ if len(targetOccurrences) > len(sourceOccurrences) || !hasArgumentSubsetMatching(sourceOccurrences, targetOccurrences, targetLocale) {
+ issues = append(issues, Issue{Code: CodeArgumentMismatch, Argument: name, Message: fmt.Sprintf("ICU exact-selector branch introduces an incompatible occurrence of argument %q", name)})
+ }
+ }
+ return issues
+}
+
+func hasArgumentSubsetMatching(source, target []*Argument, targetLocale string) bool {
+ matchedTarget := make([]int, len(source))
+ for index := range matchedTarget {
+ matchedTarget[index] = -1
+ }
+ var match func(int, []bool) bool
+ match = func(targetIndex int, visited []bool) bool {
+ for sourceIndex := range source {
+ if visited[sourceIndex] || len(compareArguments(source[sourceIndex], target[targetIndex], targetLocale)) != 0 {
+ continue
+ }
+ visited[sourceIndex] = true
+ if matchedTarget[sourceIndex] == -1 || match(matchedTarget[sourceIndex], visited) {
+ matchedTarget[sourceIndex] = targetIndex
+ return true
+ }
+ }
+ return false
+ }
+ for targetIndex := range target {
+ if !match(targetIndex, make([]bool, len(source))) {
+ return false
+ }
+ }
+ return true
+}
+
+func comparePluralSelectors(source, target *Argument, targetLocale string) []Issue {
+ name := source.Name
+ categories, err := localeid.CardinalCategories(targetLocale)
+ if source.Type == ArgumentSelectOrdinal {
+ categories, err = localeid.OrdinalCategories(targetLocale)
+ }
+ if err != nil {
+ return []Issue{{Code: CodeInvalidPluralCategory, Argument: name, Message: fmt.Sprintf("cannot resolve plural categories for locale %q: %v", targetLocale, err)}}
+ }
+ allowed := make(map[string]struct{}, len(categories))
+ for _, category := range categories {
+ allowed[category] = struct{}{}
+ }
+ sourceSelectors := selectorSet(source)
+ targetSelectors := selectorSet(target)
+ var issues []Issue
+ for selector := range targetSelectors {
+ if isExactSelector(selector) {
+ continue
+ }
+ if _, ok := allowed[selector]; !ok {
+ issues = append(issues, Issue{Code: CodeInvalidPluralCategory, Argument: name, Message: fmt.Sprintf("ICU argument %q plural category %q is not used by locale %q", name, selector, targetLocale)})
+ }
+ }
+ for selector := range sourceSelectors {
+ _, targetHasSelector := targetSelectors[selector]
+ if targetHasSelector {
+ continue
+ }
+ _, applicableCategory := allowed[selector]
+ if isExactSelector(selector) || applicableCategory {
+ issues = append(issues, Issue{Code: CodeSelectorMismatch, Argument: name, Message: fmt.Sprintf("ICU argument %q target is missing selector %q", name, selector)})
+ }
+ }
+ return issues
+}
+
+func immediateArguments(message *Message) map[string][]*Argument {
+ result := make(map[string][]*Argument)
+ for _, element := range message.elements {
+ if element.kind == elementArgument {
+ argument := element.argument
+ result[argument.Name] = append(result[argument.Name], argument)
+ }
+ }
+ return result
+}
+
+func optionsBySelector(argument *Argument) map[string]*Message {
+ result := make(map[string]*Message, len(argument.Options))
+ for _, option := range argument.Options {
+ result[option.Selector] = option.Message
+ }
+ return result
+}
+
+func selectorSet(argument *Argument) map[string]struct{} {
+ result := make(map[string]struct{}, len(argument.Options))
+ for _, option := range argument.Options {
+ result[option.Selector] = struct{}{}
+ }
+ return result
+}
+
+func sameOptionSet(left, right map[string]*Message) bool {
+ if len(left) != len(right) {
+ return false
+ }
+ for value := range left {
+ if _, ok := right[value]; !ok {
+ return false
+ }
+ }
+ return true
+}
+
+func displayType(argumentType ArgumentType) string {
+ if argumentType == ArgumentSimple {
+ return "argument"
+ }
+ return string(argumentType)
+}
+
+func isExactSelector(selector string) bool {
+ _, ok := normalizeExactSelector(selector)
+ return ok
+}
+
+func normalizeExactSelector(selector string) (string, bool) {
+ if !strings.HasPrefix(selector, "=") || len(selector) == 1 {
+ return "", false
+ }
+ canonical, ok := normalizeNumber(selector[1:], true)
+ if !ok {
+ return "", false
+ }
+ return "=" + canonical, true
+}
+
+func normalizeNumber(number string, allowNegative bool) (string, bool) {
+ if number == "" {
+ return "", false
+ }
+ negative := false
+ if number[0] == '-' {
+ if !allowNegative {
+ return "", false
+ }
+ negative = true
+ number = number[1:]
+ }
+ if number == "" {
+ return "", false
+ }
+ dotSeen := false
+ digitSeenAfterDot := false
+ for _, character := range number {
+ switch {
+ case character >= '0' && character <= '9':
+ if dotSeen {
+ digitSeenAfterDot = true
+ }
+ case character == '.' && !dotSeen:
+ dotSeen = true
+ default:
+ return "", false
+ }
+ }
+ if dotSeen && !digitSeenAfterDot {
+ return "", false
+ }
+ integer, fraction, _ := strings.Cut(number, ".")
+ integer = strings.TrimLeft(integer, "0")
+ if integer == "" {
+ integer = "0"
+ }
+ fraction = strings.TrimRight(fraction, "0")
+ if integer == "0" && fraction == "" {
+ negative = false
+ }
+ canonical := ""
+ if negative {
+ canonical += "-"
+ }
+ canonical += integer
+ if fraction != "" {
+ canonical += "." + fraction
+ }
+ return canonical, true
+}
+
+type parser struct {
+ input string
+ position int
+ depth int
+}
+
+func (p *parser) parseBranch(pluralContext bool) (*Message, error) {
+ if p.depth >= maxNestingDepth {
+ return nil, p.errorf("ICU message nesting exceeds %d levels", maxNestingDepth)
+ }
+ p.depth++
+ defer func() { p.depth-- }()
+ return p.parseMessage(true, pluralContext)
+}
+
+func (p *parser) parseMessage(nested, pluralContext bool) (*Message, error) {
+ message := &Message{}
+ var literal strings.Builder
+ flush := func() {
+ if literal.Len() == 0 {
+ return
+ }
+ message.elements = append(message.elements, element{kind: elementText, text: literal.String()})
+ literal.Reset()
+ }
+
+ for p.position < len(p.input) {
+ switch current := p.input[p.position]; current {
+ case '{':
+ flush()
+ argument, err := p.parseArgument(pluralContext)
+ if err != nil {
+ return nil, err
+ }
+ message.elements = append(message.elements, element{kind: elementArgument, argument: argument})
+ case '}':
+ if !nested {
+ return nil, p.errorf("unexpected closing brace")
+ }
+ flush()
+ return message, nil
+ case '#':
+ if pluralContext {
+ flush()
+ message.elements = append(message.elements, element{kind: elementPound})
+ p.position++
+ continue
+ }
+ literal.WriteByte(current)
+ p.position++
+ case '\'':
+ quoted, err := p.parseApostrophe(pluralContext)
+ if err != nil {
+ return nil, err
+ }
+ literal.WriteString(quoted)
+ default:
+ literal.WriteByte(current)
+ p.position++
+ }
+ }
+ if nested {
+ return nil, p.errorf("unterminated message branch")
+ }
+ flush()
+ return message, nil
+}
+
+func (p *parser) parseApostrophe(pluralContext bool) (string, error) {
+ p.position++
+ if p.position < len(p.input) && p.input[p.position] == '\'' {
+ p.position++
+ return "'", nil
+ }
+ if p.position >= len(p.input) || !p.apostropheStartsQuote(pluralContext) {
+ return "'", nil
+ }
+ var quoted strings.Builder
+ for p.position < len(p.input) {
+ if p.input[p.position] != '\'' {
+ quoted.WriteByte(p.input[p.position])
+ p.position++
+ continue
+ }
+ if p.position+1 < len(p.input) && p.input[p.position+1] == '\'' {
+ quoted.WriteByte('\'')
+ p.position += 2
+ continue
+ }
+ p.position++
+ return quoted.String(), nil
+ }
+ // ICU MessageFormat treats the end of a pattern as the implicit end of an
+ // apostrophe-quoted syntax span.
+ return quoted.String(), nil
+}
+
+func (p *parser) apostropheStartsQuote(pluralContext bool) bool {
+ switch p.input[p.position] {
+ case '{', '}':
+ return true
+ case '#':
+ return pluralContext
+ default:
+ return false
+ }
+}
+
+func (p *parser) parseArgument(pluralContext bool) (*Argument, error) {
+ p.position++
+ p.skipSpace()
+ name := p.readIdentifier()
+ if name == "" {
+ return nil, p.errorf("argument name is required")
+ }
+ p.skipSpace()
+ if p.consume('}') {
+ return &Argument{Name: name}, nil
+ }
+ if !p.consume(',') {
+ return nil, p.errorf("expected comma or closing brace after argument %q", name)
+ }
+ p.skipSpace()
+ argumentType := ArgumentType(strings.ToLower(p.readIdentifier()))
+ switch argumentType {
+ case ArgumentSelect, ArgumentPlural, ArgumentSelectOrdinal, ArgumentNumber, ArgumentDate, ArgumentTime:
+ default:
+ return nil, p.errorf("unsupported argument type %q", argumentType)
+ }
+ argument := &Argument{Name: name, Type: argumentType}
+ p.skipSpace()
+ if argumentType == ArgumentNumber || argumentType == ArgumentDate || argumentType == ArgumentTime {
+ if p.consume(',') {
+ styleStart := p.position
+ quoted := false
+ for p.position < len(p.input) {
+ current := p.input[p.position]
+ if current == '\'' {
+ if p.position+1 < len(p.input) && p.input[p.position+1] == '\'' {
+ p.position += 2
+ continue
+ }
+ quoted = !quoted
+ p.position++
+ continue
+ }
+ if current == '}' && !quoted {
+ break
+ }
+ if current == '{' && !quoted {
+ return nil, p.errorf("unexpected opening brace in %s style", argumentType)
+ }
+ p.position++
+ }
+ argument.Style = strings.TrimFunc(p.input[styleStart:p.position], func(character rune) bool {
+ return unicode.Is(unicode.Pattern_White_Space, character)
+ })
+ if argument.Style == "" {
+ return nil, p.errorf("%s style must not be empty", argumentType)
+ }
+ }
+ if !p.consume('}') {
+ return nil, p.errorf("unterminated %s argument %q", argumentType, name)
+ }
+ return argument, nil
+ }
+
+ if !p.consume(',') {
+ return nil, p.errorf("expected comma before selectors for argument %q", name)
+ }
+ p.skipSpace()
+ if argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal {
+ if strings.HasPrefix(p.input[p.position:], "offset:") {
+ p.position += len("offset:")
+ p.skipSpace()
+ start := p.position
+ for p.position < len(p.input) && (p.input[p.position] >= '0' && p.input[p.position] <= '9' || p.input[p.position] == '.') {
+ p.position++
+ }
+ offset, ok := normalizeNumber(p.input[start:p.position], false)
+ if !ok {
+ return nil, p.errorf("plural offset must be a non-negative finite number")
+ }
+ if offset != "0" {
+ argument.Offset = offset
+ }
+ p.skipSpace()
+ }
+ }
+
+ selectors := make(map[string]struct{})
+ for p.position < len(p.input) && p.input[p.position] != '}' {
+ selector := p.readSelector()
+ if selector == "" {
+ return nil, p.errorf("selector is required for argument %q", name)
+ }
+ if argumentType != ArgumentSelect {
+ if canonical, ok := normalizeExactSelector(selector); ok {
+ selector = canonical
+ }
+ }
+ if _, duplicate := selectors[selector]; duplicate {
+ return nil, p.errorf("duplicate selector %q for argument %q", selector, name)
+ }
+ if argumentType == ArgumentSelect {
+ if !isSelectKeyword(selector) {
+ return nil, p.errorf("invalid select keyword %q", selector)
+ }
+ } else if !isPluralSelector(selector) {
+ return nil, p.errorf("invalid plural selector %q", selector)
+ }
+ selectors[selector] = struct{}{}
+ p.skipSpace()
+ if !p.consume('{') {
+ return nil, p.errorf("expected message branch for selector %q", selector)
+ }
+ branchContext := pluralContext || argumentType == ArgumentPlural || argumentType == ArgumentSelectOrdinal
+ branch, err := p.parseBranch(branchContext)
+ if err != nil {
+ return nil, err
+ }
+ if !p.consume('}') {
+ return nil, p.errorf("unterminated selector %q", selector)
+ }
+ argument.Options = append(argument.Options, Option{Selector: selector, Message: branch})
+ p.skipSpace()
+ }
+ if !p.consume('}') {
+ return nil, p.errorf("unterminated argument %q", name)
+ }
+ if _, ok := selectors["other"]; !ok {
+ return nil, p.errorf("argument %q requires an \"other\" selector", name)
+ }
+ return argument, nil
+}
+
+func (p *parser) readIdentifier() string {
+ start := p.position
+ for p.position < len(p.input) {
+ width := identifierWidth(p.input, p.position)
+ if width == 0 {
+ break
+ }
+ p.position += width
+ }
+ return p.input[start:p.position]
+}
+
+func (p *parser) readSelector() string {
+ p.skipSpace()
+ start := p.position
+ for p.position < len(p.input) && p.input[p.position] != '{' && p.input[p.position] != '}' {
+ if spaceWidth(p.input, p.position) != 0 {
+ break
+ }
+ _, width := utf8.DecodeRuneInString(p.input[p.position:])
+ p.position += width
+ }
+ return p.input[start:p.position]
+}
+
+func (p *parser) skipSpace() {
+ for p.position < len(p.input) {
+ width := spaceWidth(p.input, p.position)
+ if width == 0 {
+ break
+ }
+ p.position += width
+ }
+}
+
+func (p *parser) consume(expected byte) bool {
+ if p.position >= len(p.input) || p.input[p.position] != expected {
+ return false
+ }
+ p.position++
+ return true
+}
+
+func (p *parser) errorf(format string, arguments ...any) error {
+ return fmt.Errorf("at byte %d: %s", p.position, fmt.Sprintf(format, arguments...))
+}
+
+func identifierWidth(input string, position int) int {
+ character, width := utf8.DecodeRuneInString(input[position:])
+ if character == utf8.RuneError && width == 1 {
+ return 0
+ }
+ if character < utf8.RuneSelf {
+ value := byte(character)
+ if value == '_' || value == '-' || value == '.' || value >= '0' && value <= '9' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' {
+ return width
+ }
+ return 0
+ }
+ if unicode.Is(unicode.Pattern_Syntax, character) || unicode.Is(unicode.Pattern_White_Space, character) {
+ return 0
+ }
+ return width
+}
+
+func isSelectKeyword(input string) bool {
+ if input == "" || !utf8.ValidString(input) {
+ return false
+ }
+ for _, character := range input {
+ if unicode.Is(unicode.Pattern_Syntax, character) || unicode.Is(unicode.Pattern_White_Space, character) {
+ return false
+ }
+ }
+ return true
+}
+
+func spaceWidth(input string, position int) int {
+ character, width := utf8.DecodeRuneInString(input[position:])
+ if character == utf8.RuneError && width == 1 || !unicode.Is(unicode.Pattern_White_Space, character) {
+ return 0
+ }
+ return width
+}
+
+func isPluralSelector(selector string) bool {
+ if isExactSelector(selector) {
+ return true
+ }
+ switch selector {
+ case "zero", "one", "two", "few", "many", "other":
+ return true
+ default:
+ return false
+ }
+}
+
+// String emits a deterministic ICU representation.
+func (message *Message) String() string {
+ var output strings.Builder
+ message.writeTo(&output, false)
+ return output.String()
+}
+
+func (message *Message) writeTo(output *strings.Builder, pluralContext bool) {
+ for _, element := range message.elements {
+ switch element.kind {
+ case elementText:
+ writeLiteral(output, element.text, pluralContext)
+ case elementPound:
+ output.WriteByte('#')
+ case elementArgument:
+ element.argument.writeTo(output, pluralContext)
+ }
+ }
+}
+
+func (argument *Argument) writeTo(output *strings.Builder, pluralContext bool) {
+ output.WriteByte('{')
+ output.WriteString(argument.Name)
+ if argument.Type == ArgumentSimple {
+ output.WriteByte('}')
+ return
+ }
+ output.WriteString(", ")
+ output.WriteString(string(argument.Type))
+ if argument.Type == ArgumentNumber || argument.Type == ArgumentDate || argument.Type == ArgumentTime {
+ if argument.Style != "" {
+ output.WriteString(", ")
+ output.WriteString(argument.Style)
+ }
+ output.WriteByte('}')
+ return
+ }
+ output.WriteString(", ")
+ if argument.Offset != "" {
+ _, _ = fmt.Fprintf(output, "offset:%s ", argument.Offset)
+ }
+ options := append([]Option(nil), argument.Options...)
+ sort.SliceStable(options, func(left, right int) bool { return options[left].Selector < options[right].Selector })
+ for index, option := range options {
+ if index > 0 {
+ output.WriteByte(' ')
+ }
+ output.WriteString(option.Selector)
+ output.WriteString(" {")
+ branchContext := pluralContext || argument.Type == ArgumentPlural || argument.Type == ArgumentSelectOrdinal
+ option.Message.writeTo(output, branchContext)
+ output.WriteByte('}')
+ }
+ output.WriteByte('}')
+}
+
+func writeLiteral(output *strings.Builder, literal string, pluralContext bool) {
+ characters := []rune(literal)
+ firstQuoted := -1
+ lastQuoted := -1
+ for index, character := range characters {
+ if literalNeedsQuote(character, pluralContext) {
+ if firstQuoted < 0 {
+ firstQuoted = index
+ }
+ lastQuoted = index
+ }
+ }
+ if firstQuoted < 0 {
+ writeUnquotedLiteral(output, characters)
+ return
+ }
+ writeUnquotedLiteral(output, characters[:firstQuoted])
+ output.WriteByte('\'')
+ for _, character := range characters[firstQuoted : lastQuoted+1] {
+ if character == '\'' {
+ output.WriteString("''")
+ } else {
+ output.WriteRune(character)
+ }
+ }
+ output.WriteByte('\'')
+ writeUnquotedLiteral(output, characters[lastQuoted+1:])
+}
+
+func literalNeedsQuote(character rune, pluralContext bool) bool {
+ return character == '{' || character == '}' || pluralContext && character == '#'
+}
+
+func writeUnquotedLiteral(output *strings.Builder, characters []rune) {
+ for _, character := range characters {
+ if character == '\'' {
+ output.WriteString("''")
+ } else {
+ output.WriteRune(character)
+ }
+ }
+}
diff --git a/internal/message/message_test.go b/internal/message/message_test.go
new file mode 100644
index 0000000..a9a28ba
--- /dev/null
+++ b/internal/message/message_test.go
@@ -0,0 +1,434 @@
+package message
+
+import (
+ "regexp"
+ "slices"
+ "strings"
+ "testing"
+)
+
+func TestParseAndPrintNestedICUMessage(t *testing.T) {
+ input := "{gender, select, female {{count, plural, offset:1 =0 {None} one {She and one other} other {She and # others}}} other {{count, number, ::compact-short} items}}"
+ parsed, err := Parse(input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ printed := parsed.String()
+ reparsed, err := Parse(printed)
+ if err != nil {
+ t.Fatalf("printed message is invalid: %v\n%s", err, printed)
+ }
+ if reparsed.String() != printed {
+ t.Fatalf("printer is unstable:\nfirst: %s\nsecond: %s", printed, reparsed.String())
+ }
+}
+
+func TestParseRejectsMalformedMessages(t *testing.T) {
+ tests := []string{
+ "{count, plural, one {One}",
+ "{count, plural, one {One}}",
+ "{count, plural other {Items}}",
+ "{kind, select, other {One} other {Two}}",
+ "{count, plural, offset:nope other {Items}}",
+ }
+ for _, input := range tests {
+ t.Run(input, func(t *testing.T) {
+ if _, err := Parse(input); err == nil {
+ t.Fatalf("Parse(%q) succeeded", input)
+ }
+ })
+ }
+}
+
+func TestParseRejectsExcessiveNesting(t *testing.T) {
+ input := strings.Repeat("{value, select, other {", maxNestingDepth+1) + "value" + strings.Repeat("}}", maxNestingDepth+1)
+ if _, err := Parse(input); err == nil || !strings.Contains(err.Error(), "nesting exceeds") {
+ t.Fatalf("Parse error = %v, want nesting limit error", err)
+ }
+}
+
+func TestCompareChecksArgumentsAndSelectors(t *testing.T) {
+ issues := Compare(
+ "{gender, select, female {{count, plural, one {One} other {#}}} other {They}}",
+ "{gender, select, female {{count, select, one {Un} other {Plusieurs}}} male {Il} other {Iel}}",
+ "fr",
+ )
+ got := make([]Code, len(issues))
+ for index, issue := range issues {
+ got[index] = issue.Code
+ }
+ if !slices.Contains(got, CodeSelectorMismatch) || !slices.Contains(got, CodeArgumentTypeMismatch) {
+ t.Fatalf("Compare issues = %#v", issues)
+ }
+}
+
+func TestPreservedTextTokenKindsCompareCorrespondingPluralBranches(t *testing.T) {
+ tagPattern := regexp.MustCompile(``)
+ codePattern := regexp.MustCompile("`[^`]+`")
+ tokenizers := []func(string) []string{
+ func(input string) []string { return tagPattern.FindAllString(input, -1) },
+ func(input string) []string { return codePattern.FindAllString(input, -1) },
+ }
+ source := "{n, plural, one {{name} `go test`} other {{name} `go test`}}"
+ target := "{n, plural, one {{name} `go test`} few {{name} `go test`} many {{name} `go test`} other {{name} `go test`}}"
+ preserved, err := PreservedTextTokenKinds(source, target, "ru", tokenizers)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(preserved, []bool{true, true}) {
+ t.Fatalf("PreservedTextTokenKinds() = %v, want both token kinds preserved", preserved)
+ }
+
+ damaged := "{n, plural, one {{name} `go test`} few {{name} `go test`} many {{name} `go test`} other {{name} `go test`}}"
+ preserved, err = PreservedTextTokenKinds(source, damaged, "ru", tokenizers)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(preserved, []bool{false, true}) {
+ t.Fatalf("PreservedTextTokenKinds() = %v, want only tag mismatch", preserved)
+ }
+ if _, err := PreservedTextTokenKinds("{broken", target, "ru", tokenizers); err == nil {
+ t.Fatal("PreservedTextTokenKinds accepted malformed source")
+ }
+ if _, err := PreservedTextTokenKinds(source, "{broken", "ru", tokenizers); err == nil {
+ t.Fatal("PreservedTextTokenKinds accepted malformed target")
+ }
+}
+
+func TestCompareRejectsLocaleInapplicablePluralCategory(t *testing.T) {
+ issues := Compare(
+ "{count, plural, one {One} other {# items}}",
+ "{count, plural, one {ఒక అంశం} few {కొన్ని అంశాలు} other {# అంశాలు}}",
+ "te",
+ )
+ if !hasCode(issues, CodeInvalidPluralCategory) {
+ t.Fatalf("Compare issues = %#v, want %q", issues, CodeInvalidPluralCategory)
+ }
+}
+
+func TestCompareAllowsLocaleCategoryAndExactSelector(t *testing.T) {
+ issues := Compare(
+ "{count, plural, one {One} other {# items}}",
+ "{count, plural, =0 {Нет элементов} one {Один элемент} few {# элемента} other {# элементов}}",
+ "ru",
+ )
+ if len(issues) != 0 {
+ t.Fatalf("Compare returned issues for valid Russian message: %#v", issues)
+ }
+}
+
+func TestCompareRejectsArgumentIntroducedByTargetOnlyExactSelector(t *testing.T) {
+ issues := Compare(
+ "{n, plural, other {# items}}",
+ "{n, plural, =0 {{price, number}} other {# articles}}",
+ "fr",
+ )
+ if !hasCode(issues, CodeArgumentMismatch) {
+ t.Fatalf("Compare issues = %#v, want target-only exact argument mismatch", issues)
+ }
+}
+
+func TestParseRejectsNonFiniteAndMalformedExactSelectors(t *testing.T) {
+ for _, selector := range []string{"NaN", "Inf", "Infinity", "1.", "0x1p2"} {
+ t.Run(selector, func(t *testing.T) {
+ if _, err := Parse("{n, plural, =" + selector + " {bad} other {ok}}"); err == nil {
+ t.Fatalf("Parse accepted invalid exact selector %q", selector)
+ }
+ })
+ }
+ for _, selector := range []string{"0", "-1", "1.5"} {
+ t.Run("valid_"+selector, func(t *testing.T) {
+ if _, err := Parse("{n, plural, =" + selector + " {ok} other {other}}"); err != nil {
+ t.Fatalf("Parse rejected valid exact selector %q: %v", selector, err)
+ }
+ })
+ }
+}
+
+func TestCompareNormalizesEquivalentExactSelectors(t *testing.T) {
+ source := "{n, plural, =1 {One} other {# items}}"
+ for _, selector := range []string{"1.0", "01", "01.000"} {
+ t.Run(selector, func(t *testing.T) {
+ target := "{n, plural, =" + selector + " {Un} other {# articles}}"
+ if issues := Compare(source, target, "fr"); len(issues) != 0 {
+ t.Fatalf("Compare rejected equivalent selector %q: %#v", selector, issues)
+ }
+ })
+ }
+ if _, err := Parse("{n, plural, =1 {One} =1.0 {Duplicate} other {Other}}"); err == nil {
+ t.Fatal("Parse accepted numerically duplicate exact selectors")
+ }
+}
+
+func TestParseImplicitlyClosesApostropheQuoteAtEnd(t *testing.T) {
+ parsed, err := Parse("Hello {name}: '{")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Parse(parsed.String()); err != nil {
+ t.Fatalf("Parse(String()) failed: %v", err)
+ }
+}
+
+func TestParseAllowsWhitespaceAfterPluralOffsetColon(t *testing.T) {
+ if _, err := Parse("{n, plural, offset: 1 one {One} other {# items}}"); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestParsePreservesDecimalPluralOffsets(t *testing.T) {
+ parsed, err := Parse("{n, plural, offset:00.500 one {One} other {# items}}")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := parsed.String(); !strings.Contains(got, "offset:0.5 ") {
+ t.Fatalf("String() = %q, want canonical decimal offset", got)
+ }
+ if issues := Compare(
+ "{n, plural, offset:0.5 one {One} other {# items}}",
+ "{n, plural, offset:00.500 one {Un} other {# articles}}",
+ "fr",
+ ); len(issues) != 0 {
+ t.Fatalf("Compare rejected equivalent decimal offsets: %#v", issues)
+ }
+ if issues := Compare(
+ "{n, plural, offset:0.5 one {One} other {# items}}",
+ "{n, plural, offset:1 one {Un} other {# articles}}",
+ "fr",
+ ); !hasCode(issues, CodeSelectorMismatch) {
+ t.Fatalf("Compare issues = %#v, want decimal offset mismatch", issues)
+ }
+}
+
+func TestParseUsesICUPatternWhitespace(t *testing.T) {
+ if _, err := Parse("{\u200ename\u200e,\u200eplural\u200e,\u200eone {One} other {Other}}"); err != nil {
+ t.Fatalf("Parse rejected Pattern_White_Space separator: %v", err)
+ }
+ parsed, err := Parse("{\u00a0name, plural, one {One} other {Other}}")
+ if err != nil {
+ t.Fatalf("Parse rejected NBSP as part of an ICU identifier: %v", err)
+ }
+ if got := parsed.String(); !strings.HasPrefix(got, "{\u00a0name,") {
+ t.Fatalf("String() = %q, NBSP was incorrectly consumed as syntax whitespace", got)
+ }
+}
+
+func TestParseAllowsApostropheQuotedBracesInFormatterStyles(t *testing.T) {
+ input := "{d, date, '{'yyyy'}'}"
+ parsed, err := Parse(input)
+ if err != nil {
+ t.Fatalf("Parse rejected quoted formatter braces: %v", err)
+ }
+ if got := parsed.String(); got != "{d, date, '{'yyyy'}'}" {
+ t.Fatalf("String() = %q, want quoted formatter style preserved", got)
+ }
+}
+
+func TestLooksLikeTypedICUArgumentAfterPercentLiteral(t *testing.T) {
+ source := "%{amount, number}"
+ if !LooksLike(source) {
+ t.Fatalf("LooksLike(%q) = false", source)
+ }
+ if issues := Compare(source, "%", "fr"); !hasCode(issues, CodeArgumentMismatch) {
+ t.Fatalf("Compare issues = %#v, want missing typed argument", issues)
+ }
+ if LooksLike("%{amount}") {
+ t.Fatal("LooksLike treated legacy percent interpolation as ICU")
+ }
+}
+
+func TestComparePreservesNonPatternWhitespaceInFormatterStyles(t *testing.T) {
+ source := "{d, date,\u00a0yyyy}"
+ target := "{d, date, yyyy}"
+ parsed, err := Parse(source)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := parsed.String(); !strings.Contains(got, "\u00a0yyyy") {
+ t.Fatalf("String() = %q, want NBSP preserved", got)
+ }
+ if issues := Compare(source, target, "fr"); !hasCode(issues, CodeArgumentStyleMismatch) {
+ t.Fatalf("Compare issues = %#v, want formatter style mismatch", issues)
+ }
+}
+
+func TestParseValidatesSelectKeywords(t *testing.T) {
+ if _, err := Parse("{g, select, male,female {x} other {y}}"); err == nil {
+ t.Fatal("Parse accepted punctuation in select keyword")
+ }
+ for _, keyword := range []string{"male-female", "admin.user"} {
+ if _, err := Parse("{g, select, " + keyword + " {x} other {y}}"); err == nil {
+ t.Fatalf("Parse accepted Pattern_Syntax in select keyword %q", keyword)
+ }
+ }
+ if _, err := Parse("{g, select, мужчина {x} other {y}}"); err != nil {
+ t.Fatalf("Parse rejected Unicode select keyword: %v", err)
+ }
+}
+
+func TestCompareChecksTargetOnlyLocaleCategoryAgainstSourceOther(t *testing.T) {
+ source := "{count, plural, one {{name} has one item} other {{name} has # items}}"
+ valid := "{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} other {{name} имеет # предметов}}"
+ if issues := Compare(source, valid, "ru"); len(issues) != 0 {
+ t.Fatalf("Compare rejected structurally valid target-only category: %#v", issues)
+ }
+
+ invalid := "{count, plural, one {{name} имеет один предмет} few {несколько предметов} other {{name} имеет # предметов}}"
+ issues := Compare(source, invalid, "ru")
+ if !hasCode(issues, CodeArgumentMismatch) || !hasCode(issues, CodeSelectorMismatch) {
+ t.Fatalf("Compare issues = %#v, want missing argument and pound placeholder", issues)
+ }
+}
+
+func TestComparePairsRepeatedPluralsWithTargetOnlyCategories(t *testing.T) {
+ source := "{n, plural, one {{a}} other {{a} #}} then {n, plural, one {{z}} other {{z} #}}"
+ target := "{n, plural, one {{z}} few {{z} #} other {{z} #}} puis {n, plural, one {{a}} other {{a} #}}"
+ if issues := Compare(source, target, "ru"); len(issues) != 0 {
+ t.Fatalf("Compare rejected valid repeated plural reordering: %#v", issues)
+ }
+}
+
+func TestComparePreservesPluralOffsetsAndFormatterStyles(t *testing.T) {
+ tests := []struct {
+ name string
+ source string
+ target string
+ code Code
+ }{
+ {
+ name: "plural offset",
+ source: "{count, plural, offset:1 one {One} other {#}}",
+ target: "{count, plural, one {Un} other {#}}",
+ code: CodeSelectorMismatch,
+ },
+ {
+ name: "number style",
+ source: "Total: {amount, number, ::currency/USD}",
+ target: "Total : {amount, number, ::percent}",
+ code: CodeArgumentStyleMismatch,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ issues := Compare(test.source, test.target, "fr")
+ if !hasCode(issues, test.code) {
+ t.Fatalf("Compare issues = %#v, want %q", issues, test.code)
+ }
+ })
+ }
+}
+
+func TestComparePreservesArgumentsWithinCorrespondingBranches(t *testing.T) {
+ source := "{gender, select, female {{count, plural, offset:1 one {One} other {#}}} other {{count, plural, one {One} other {#}}}}"
+ target := "{gender, select, female {{count, plural, one {Une} other {#}}} other {{count, plural, offset:1 one {Une} other {#}}}}"
+ issues := Compare(source, target, "fr")
+ if !hasCode(issues, CodeSelectorMismatch) {
+ t.Fatalf("Compare issues = %#v, want branch-specific offset mismatch", issues)
+ }
+}
+
+func TestCompareAllowsRepeatedArgumentReorderingWithinBranch(t *testing.T) {
+ source := "Created {date, date, short}; archived {date, date, long}"
+ target := "Archivé {date, date, long}; créé {date, date, short}"
+ if issues := Compare(source, target, "fr"); len(issues) != 0 {
+ t.Fatalf("Compare rejected valid repeated-argument reordering: %#v", issues)
+ }
+}
+
+func TestCompareAllowsRepeatedNestedArgumentReorderingWithinBranch(t *testing.T) {
+ source := "{choice, select, first {{first}} other {A}} then {choice, select, first {{last}} other {B}}"
+ target := "{choice, select, first {{last}} other {B traduit}} puis {choice, select, first {{first}} other {A traduit}}"
+ if issues := Compare(source, target, "fr"); len(issues) != 0 {
+ t.Fatalf("Compare rejected valid repeated nested-argument reordering: %#v", issues)
+ }
+}
+
+func TestComparePreservesPoundPlaceholders(t *testing.T) {
+ issues := Compare(
+ "{count, plural, one {One item} other {# items}}",
+ "{count, plural, one {Un article} other {articles}}",
+ "fr",
+ )
+ if !hasCode(issues, CodeSelectorMismatch) {
+ t.Fatalf("Compare issues = %#v, want missing pound placeholder", issues)
+ }
+}
+
+func TestComparePreservesPluralPoundThroughNestedSelect(t *testing.T) {
+ source := "{n, plural, other {{gender, select, male {# men} other {# people}}}}"
+ target := "{n, plural, other {{gender, select, male {hommes} other {# personnes}}}}"
+ issues := Compare(source, target, "fr")
+ if !hasCode(issues, CodeSelectorMismatch) {
+ t.Fatalf("Compare issues = %#v, want nested missing pound placeholder", issues)
+ }
+}
+
+func TestCompareSupportsUnicodeArgumentNames(t *testing.T) {
+ source := "Bienvenue, {имя}"
+ target := "Bienvenue"
+ if !LooksLike(source) {
+ t.Fatalf("LooksLike(%q) = false", source)
+ }
+ issues := Compare(source, target, "fr")
+ if !hasCode(issues, CodeArgumentMismatch) {
+ t.Fatalf("Compare issues = %#v, want missing Unicode argument", issues)
+ }
+
+ parsed, err := Parse("{имя, select, admin {Привет, {имя}} other {Здравствуйте, {имя}}}")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Parse(parsed.String()); err != nil {
+ t.Fatalf("Parse(String()) failed: %v", err)
+ }
+}
+
+func TestCompareReportsMalformedSourceAndTarget(t *testing.T) {
+ for name, values := range map[string][2]string{
+ "source": {"{count, plural, one {One}}", "{count, plural, one {Un} other {Autres}}"},
+ "target": {"{count, plural, one {One} other {Other}}", "{count, plural, one {Un} other {Autres}"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ issues := Compare(values[0], values[1], "fr")
+ if !hasCode(issues, CodeSyntax) {
+ t.Fatalf("Compare issues = %#v, want syntax issue", issues)
+ }
+ })
+ }
+}
+
+func hasCode(issues []Issue, code Code) bool {
+ for _, issue := range issues {
+ if issue.Code == code {
+ return true
+ }
+ }
+ return false
+}
+
+func FuzzParsePrint(f *testing.F) {
+ for _, seed := range []string{
+ "plain text",
+ "Hello {name}",
+ "{gender, select, female {She} other {They}}",
+ "{count, plural, offset:1 =0 {None} one {One} other {# items}}",
+ "{place, selectordinal, one {#st} two {#nd} few {#rd} other {#th}}",
+ "This '{' is literal and this apostrophe isn''t ambiguous",
+ } {
+ f.Add(seed)
+ }
+ f.Fuzz(func(t *testing.T, input string) {
+ parsed, err := Parse(input)
+ if err != nil {
+ return
+ }
+ printed := parsed.String()
+ reparsed, err := Parse(printed)
+ if err != nil {
+ t.Fatalf("Parse(String(Parse(%q))) failed: %v; printed %q", input, err, printed)
+ }
+ if reparsed.String() != printed {
+ t.Fatalf("unstable print for %q: first %q, second %q", input, printed, reparsed.String())
+ }
+ })
+}
diff --git a/internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26 b/internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26
new file mode 100644
index 0000000..c880784
--- /dev/null
+++ b/internal/message/testdata/fuzz/FuzzParsePrint/87dbf68552061a26
@@ -0,0 +1,2 @@
+go test fuzz v1
+string("0'{''{'0")
diff --git a/internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738 b/internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738
new file mode 100644
index 0000000..25d012b
--- /dev/null
+++ b/internal/message/testdata/fuzz/FuzzParsePrint/b429763ebca9b738
@@ -0,0 +1,2 @@
+go test fuzz v1
+string("'{{'")
diff --git a/internal/policy/policy.go b/internal/policy/policy.go
index 3606505..84e72ae 100644
--- a/internal/policy/policy.go
+++ b/internal/policy/policy.go
@@ -5,6 +5,7 @@ import (
"github.com/Tom-R-Main/Internationalizer/internal/config"
"github.com/Tom-R-Main/Internationalizer/internal/glossary"
"github.com/Tom-R-Main/Internationalizer/internal/llm"
+ "github.com/Tom-R-Main/Internationalizer/internal/locale"
"github.com/Tom-R-Main/Internationalizer/internal/state"
)
@@ -22,10 +23,23 @@ type Resolved struct {
// already-loaded style-guide and glossary content so policy resolution remains
// independent of filesystem layout.
func Resolve(cfg *config.Config, targetLocale, format, styleGuide string, terms []glossary.Term) (Resolved, error) {
- effectiveLLM := cfg.LLMForLocale(targetLocale)
- prompt := llm.BuildSystemPrompt(cfg.SourceLocale, targetLocale, styleGuide, terms)
+ sourceLocale := cfg.SourceLocale
+ if sourceLocale == "" {
+ sourceLocale = "en"
+ }
+ canonicalSource, err := locale.Canonical(sourceLocale)
+ if err != nil {
+ return Resolved{}, err
+ }
+ canonicalTarget, err := locale.Canonical(targetLocale)
+ if err != nil {
+ return Resolved{}, err
+ }
+
+ effectiveLLM := cfg.LLMForLocale(canonicalTarget)
+ prompt := llm.BuildSystemPrompt(canonicalSource, canonicalTarget, styleGuide, terms)
if format == "markdown" {
- prompt = llm.BuildDocumentPrompt(cfg.SourceLocale, targetLocale, styleGuide, terms)
+ prompt = llm.BuildDocumentPrompt(canonicalSource, canonicalTarget, styleGuide, terms)
}
hash, err := state.HashValue(struct {
@@ -37,7 +51,7 @@ func Resolve(cfg *config.Config, targetLocale, format, styleGuide string, terms
Model string `json:"model"`
Reasoning string `json:"reasoning_effort"`
Prompt string `json:"prompt"`
- }{promptPolicyVersion, cfg.SourceLocale, targetLocale, format, effectiveLLM.Provider, effectiveLLM.Model, llm.EffectiveReasoningEffort(effectiveLLM), prompt})
+ }{promptPolicyVersion, canonicalSource, canonicalTarget, format, effectiveLLM.Provider, effectiveLLM.Model, llm.EffectiveReasoningEffort(effectiveLLM), prompt})
if err != nil {
return Resolved{}, err
}
diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go
index f020c09..e602acc 100644
--- a/internal/policy/policy_test.go
+++ b/internal/policy/policy_test.go
@@ -140,6 +140,25 @@ func TestResolveUsesLocaleLLMOverride(t *testing.T) {
}
}
+func TestResolveCanonicalizesLocaleIdentity(t *testing.T) {
+ canonicalConfig := baseConfig()
+ canonicalConfig.SourceLocale = "en-US"
+ canonical, err := policy.Resolve(canonicalConfig, "pt-BR", "json", "", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ variantConfig := baseConfig()
+ variantConfig.SourceLocale = "en-us"
+ variant, err := policy.Resolve(variantConfig, "pt-br", "json", "", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if variant.Hash != canonical.Hash || variant.Prompt != canonical.Prompt {
+ t.Fatal("canonical-equivalent locales produced different translation policies")
+ }
+}
+
func baseConfig() *config.Config {
return &config.Config{
SourceLocale: "en",
diff --git a/internal/state/manifest.go b/internal/state/manifest.go
index 9910543..a931405 100644
--- a/internal/state/manifest.go
+++ b/internal/state/manifest.go
@@ -9,6 +9,8 @@ import (
"os"
"path/filepath"
"time"
+
+ "github.com/Tom-R-Main/Internationalizer/internal/locale"
)
const SchemaVersion = 1
@@ -62,6 +64,16 @@ func Load(path string) (*Manifest, error) {
if manifest.Translations == nil {
manifest.Translations = make(map[string]Entry)
}
+ canonicalTranslations := make(map[string]Entry, len(manifest.Translations))
+ for _, entry := range manifest.Translations {
+ entry.Locale = canonicalLocaleOrOriginal(entry.Locale)
+ identity := Identity(entry.Bundle, entry.Key, entry.Locale)
+ if existing, duplicate := canonicalTranslations[identity]; duplicate && existing != entry {
+ return nil, fmt.Errorf("manifest %s contains conflicting entries for %s/%s/%s", path, entry.Bundle, entry.Key, entry.Locale)
+ }
+ canonicalTranslations[identity] = entry
+ }
+ manifest.Translations = canonicalTranslations
return &manifest, nil
}
@@ -85,12 +97,21 @@ func (m *Manifest) Set(entry Entry) {
if m.Translations == nil {
m.Translations = make(map[string]Entry)
}
+ entry.Locale = canonicalLocaleOrOriginal(entry.Locale)
m.Translations[Identity(entry.Bundle, entry.Key, entry.Locale)] = entry
}
// Identity returns a stable full SHA-256 identifier for a logical translation.
func Identity(bundle, key, locale string) string {
- return hashBytes([]byte(bundle + "\x00" + key + "\x00" + locale))
+ return hashBytes([]byte(bundle + "\x00" + key + "\x00" + canonicalLocaleOrOriginal(locale)))
+}
+
+func canonicalLocaleOrOriginal(value string) string {
+ canonical, err := locale.Canonical(value)
+ if err != nil {
+ return value
+ }
+ return canonical
}
// SourceHash binds translation reuse to both content and source format.
diff --git a/internal/state/manifest_test.go b/internal/state/manifest_test.go
index 64f7146..eafc7b9 100644
--- a/internal/state/manifest_test.go
+++ b/internal/state/manifest_test.go
@@ -50,6 +50,33 @@ func TestLoadRejectsMalformedManifest(t *testing.T) {
}
}
+func TestLoadMigratesLegacyLocaleIdentityInMemory(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "manifest.json")
+ data := []byte(`{"schema_version":1,"translations":{"legacy-raw-key":{"bundle":"app","key":"save","locale":"pt-br","source_hash":"source","policy_hash":"policy","target_hash":"target","updated_at":"2026-01-01T00:00:00Z"}}}`)
+ if err := os.WriteFile(path, data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ manifest, err := Load(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ entry, ok := manifest.Get("app", "save", "pt-BR")
+ if !ok || entry.Locale != "pt-BR" {
+ t.Fatalf("legacy entry = %#v, %v; want canonicalized lookup", entry, ok)
+ }
+}
+
+func TestLoadRejectsConflictingCanonicalLocaleIdentities(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "manifest.json")
+ data := []byte(`{"schema_version":1,"translations":{"first":{"bundle":"app","key":"save","locale":"pt-br","source_hash":"one","updated_at":"2026-01-01T00:00:00Z"},"second":{"bundle":"app","key":"save","locale":"pt-BR","source_hash":"two","updated_at":"2026-01-01T00:00:00Z"}}}`)
+ if err := os.WriteFile(path, data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Load(path); err == nil {
+ t.Fatal("Load accepted conflicting canonical-equivalent identities")
+ }
+}
+
func TestHashesSeparateSourceFormatAndPolicy(t *testing.T) {
if SourceHash("json", "Save") == SourceHash("markdown", "Save") {
t.Fatal("source hash ignored format")
@@ -59,6 +86,23 @@ func TestHashesSeparateSourceFormatAndPolicy(t *testing.T) {
}
}
+func TestManifestIdentityCanonicalizesLocale(t *testing.T) {
+ manifest := New()
+ entry := Entry{Bundle: "app", Key: "save", Locale: "pt-br"}
+ manifest.Set(entry)
+
+ got, ok := manifest.Get("app", "save", "pt-BR")
+ if !ok {
+ t.Fatal("canonical-equivalent locale did not find the manifest entry")
+ }
+ if got.Locale != "pt-BR" {
+ t.Fatalf("stored locale = %q, want %q", got.Locale, "pt-BR")
+ }
+ if Identity("app", "save", "pt-br") != Identity("app", "save", "pt-BR") {
+ t.Fatal("canonical-equivalent locales produced different identities")
+ }
+}
+
func TestHashValueRejectsUnsupportedValue(t *testing.T) {
if _, err := HashValue(func() {}); err == nil {
t.Fatal("HashValue accepted an unsupported value")
diff --git a/internal/tm/tm.go b/internal/tm/tm.go
index d698aa9..48eb151 100644
--- a/internal/tm/tm.go
+++ b/internal/tm/tm.go
@@ -10,6 +10,8 @@ import (
"path/filepath"
"sync"
"time"
+
+ "github.com/Tom-R-Main/Internationalizer/internal/locale"
)
// Record is a single translation memory entry.
@@ -69,6 +71,7 @@ func Load(path string) (*TM, error) {
if err := json.Unmarshal(line, &rec); err != nil {
return nil, fmt.Errorf("parsing TM %s line %d: %w", path, lineNumber, err)
}
+ rec = canonicalizeRecord(rec)
if t.index[rec.Locale] == nil {
t.index[rec.Locale] = make(map[string]Record)
}
@@ -83,7 +86,7 @@ func (t *TM) Lookup(locale, bundle, key, sourceHash, policyHash string) (Record,
t.mu.RLock()
defer t.mu.RUnlock()
- localeMap, ok := t.index[locale]
+ localeMap, ok := t.index[canonicalLocaleOrOriginal(locale)]
if !ok {
return Record{}, false
}
@@ -98,6 +101,7 @@ func (t *TM) Lookup(locale, bundle, key, sourceHash, policyHash string) (Record,
func (t *TM) Add(rec Record) error {
t.mu.Lock()
defer t.mu.Unlock()
+ rec = canonicalizeRecord(rec)
if err := os.MkdirAll(filepath.Dir(t.path), 0o755); err != nil {
return fmt.Errorf("creating TM directory: %w", err)
@@ -138,6 +142,9 @@ func (t *TM) AddBatch(records []Record) error {
t.mu.Lock()
defer t.mu.Unlock()
+ for i := range records {
+ records[i] = canonicalizeRecord(records[i])
+ }
if err := os.MkdirAll(filepath.Dir(t.path), 0o755); err != nil {
return fmt.Errorf("creating TM directory: %w", err)
@@ -273,3 +280,16 @@ func HashSource(source string) string {
func recordKey(bundle, key string) string {
return bundle + "\x00" + key
}
+
+func canonicalizeRecord(rec Record) Record {
+ rec.Locale = canonicalLocaleOrOriginal(rec.Locale)
+ return rec
+}
+
+func canonicalLocaleOrOriginal(value string) string {
+ canonical, err := locale.Canonical(value)
+ if err != nil {
+ return value
+ }
+ return canonical
+}
diff --git a/internal/tm/tm_test.go b/internal/tm/tm_test.go
index b3fe19a..e53c9c6 100644
--- a/internal/tm/tm_test.go
+++ b/internal/tm/tm_test.go
@@ -83,6 +83,33 @@ func TestTMLookupRequiresMatchingPolicy(t *testing.T) {
}
}
+func TestTMLocaleIdentityIsCanonical(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "tm.jsonl")
+ memory, err := Load(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ record := Record{Bundle: "app", Key: "save", Source: "Save", Target: "Salvar", Locale: "pt-br", Hash: HashSource("Save"), PolicyHash: "policy-v1", Timestamp: time.Now()}
+ if err := memory.Add(record); err != nil {
+ t.Fatal(err)
+ }
+ got, ok := memory.Lookup("pt-BR", "app", "save", HashSource("Save"), "policy-v1")
+ if !ok {
+ t.Fatal("canonical-equivalent locale did not reuse the translation")
+ }
+ if got.Locale != "pt-BR" {
+ t.Fatalf("stored locale = %q, want %q", got.Locale, "pt-BR")
+ }
+
+ reloaded, err := Load(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, ok := reloaded.Lookup("pt-BR", "app", "save", HashSource("Save"), "policy-v1"); !ok {
+ t.Fatal("canonical-equivalent locale did not reuse the translation after reload")
+ }
+}
+
func TestLoadRejectsMalformedRecord(t *testing.T) {
path := filepath.Join(t.TempDir(), "tm.jsonl")
if err := os.WriteFile(path, []byte("{\n"), 0o644); err != nil {
diff --git a/internal/translate/protected.go b/internal/translate/protected.go
index 345f58e..b3914bb 100644
--- a/internal/translate/protected.go
+++ b/internal/translate/protected.go
@@ -10,11 +10,14 @@ import (
// validateTranslationValue enforces deterministic invariants at both provider
// and explicit-adoption boundaries. Instructions in a prompt are not proof
// that protected source structure survived translation.
-func validateTranslationValue(key, source, target string) error {
+func validateTranslationValue(key, source, target, targetLocale string) error {
if strings.TrimSpace(source) != "" && strings.TrimSpace(target) == "" {
return fmt.Errorf("blank translation for %q", key)
}
- if findings := validation.ProtectedFindings(key, source, target); len(findings) > 0 {
+ if findings := validation.ProtectedFindings(key, source, target, targetLocale); len(findings) > 0 {
+ return fmt.Errorf("%s for %q", findings[0].Message, key)
+ }
+ if findings := validation.ICUFindings(key, source, target, targetLocale); len(findings) > 0 {
return fmt.Errorf("%s for %q", findings[0].Message, key)
}
return nil
diff --git a/internal/translate/protected_test.go b/internal/translate/protected_test.go
index 4226424..9574c72 100644
--- a/internal/translate/protected_test.go
+++ b/internal/translate/protected_test.go
@@ -5,7 +5,7 @@ import "testing"
func TestValidateTranslationValuePreservesProtectedMarkdown(t *testing.T) {
source := "Read [the guide](https://example.com/guide) and run `go test`.\n\n```go\nfmt.Println(\"ok\")\n```\n"
valid := "Lisez [le guide](https://example.com/guide) et lancez `go test`.\n\n```go\nfmt.Println(\"ok\")\n```\n"
- if err := validateTranslationValue("_content", source, valid); err != nil {
+ if err := validateTranslationValue("_content", source, valid, "fr"); err != nil {
t.Fatalf("valid protected Markdown was rejected: %v", err)
}
@@ -16,9 +16,17 @@ func TestValidateTranslationValuePreservesProtectedMarkdown(t *testing.T) {
}
for name, target := range tests {
t.Run(name, func(t *testing.T) {
- if err := validateTranslationValue("_content", source, target); err == nil {
+ if err := validateTranslationValue("_content", source, target, "fr"); err == nil {
t.Fatal("invalid protected Markdown was accepted")
}
})
}
}
+
+func TestValidateTranslationValueAllowsTargetOnlyICUPluralBranches(t *testing.T) {
+ source := `{count, plural, one {{name} has one item} other {{name} has # items}}`
+ target := `{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} many {{name} имеет # предметов} other {{name} имеет # предмета}}`
+ if err := validateTranslationValue("items", source, target, "ru"); err != nil {
+ t.Fatalf("valid target-only ICU branches were rejected: %v", err)
+ }
+}
diff --git a/internal/translate/translate.go b/internal/translate/translate.go
index 8db397e..0b100b0 100644
--- a/internal/translate/translate.go
+++ b/internal/translate/translate.go
@@ -111,15 +111,12 @@ func Run(ctx context.Context, cfg *config.Config, provider llm.Provider, opts Op
locales := cfg.TargetLocales
if len(opts.Locales) > 0 {
- configured := make(map[string]struct{}, len(cfg.TargetLocales))
- for _, locale := range cfg.TargetLocales {
- configured[locale] = struct{}{}
- }
seen := make(map[string]struct{}, len(opts.Locales))
locales = make([]string, 0, len(opts.Locales))
- for _, locale := range opts.Locales {
- if _, ok := configured[locale]; !ok {
- return nil, fmt.Errorf("locale %q is not in target_locales", locale)
+ for _, requested := range opts.Locales {
+ locale, ok := cfg.ConfiguredTargetLocale(requested)
+ if !ok {
+ return nil, fmt.Errorf("locale %q is not in target_locales", requested)
}
if _, ok := seen[locale]; ok {
continue
@@ -159,7 +156,7 @@ func Run(ctx context.Context, cfg *config.Config, provider llm.Provider, opts Op
currentProvider := provider
if localeProvider, ok := opts.LocaleProviders[current.locale]; ok {
currentProvider = localeProvider
- } else if _, hasOverride := cfg.LLM.LocaleOverrides[current.locale]; hasOverride {
+ } else if cfg.HasLLMOverrideForLocale(current.locale) {
currentProvider = nil
}
outputs[current.index] = translateLocale(ctx, cfg, current.bundle, currentProvider, memory, manifest, current.locale, batchSize, opts)
@@ -259,6 +256,14 @@ func translateLocale(
}
}
result := Result{Bundle: bundle.bundle.ID, Locale: locale, KeysTotal: len(sourceKeys)}
+ for _, key := range sortedKeys(sourceKeys) {
+ if findings := validation.ICUSourceFindings(key, sourceKeys[key], cfg.SourceLocale); len(findings) > 0 {
+ result.Errors = append(result.Errors, fmt.Sprintf("source %q: %s", key, findings[0].Message))
+ }
+ }
+ if len(result.Errors) > 0 {
+ return jobOutput{result: result}
+ }
targetPath, err := bundle.bundle.TargetPath(locale)
if err != nil {
result.Errors = append(result.Errors, err.Error())
@@ -320,7 +325,7 @@ func translateLocale(
if !plan.state.adoptable() {
continue
}
- if err := validateTranslationValue(plan.key, plan.source, targetKeys[plan.key]); err != nil {
+ if err := validateTranslationValue(plan.key, plan.source, targetKeys[plan.key], locale); err != nil {
result.Errors = append(result.Errors, fmt.Sprintf("cannot adopt %q: %v", plan.key, err))
}
}
@@ -355,10 +360,12 @@ func translateLocale(
toTranslate := make([]plannedEntry, 0, len(candidates))
for _, plan := range candidates {
if record, ok := memory.Lookup(locale, bundle.bundle.ID, plan.key, plan.sourceHash, policyHash); ok {
- staged[plan.key] = record.Target
- origins[plan.key] = translationOrigin{kind: "tm", provider: record.Provider, model: record.Model}
- result.KeysCached++
- continue
+ if err := validateTranslationValue(plan.key, plan.source, record.Target, locale); err == nil {
+ staged[plan.key] = record.Target
+ origins[plan.key] = translationOrigin{kind: "tm", provider: record.Provider, model: record.Model}
+ result.KeysCached++
+ continue
+ }
}
toTranslate = append(toTranslate, plan)
}
@@ -390,7 +397,7 @@ func translateLocale(
result.Errors = append(result.Errors, fmt.Sprintf("batch %d: provider returned no response", i/batchSize+1))
return jobOutput{result: result}
}
- if err := validateBatch(entries, response.Translations); err != nil {
+ if err := validateBatch(entries, response.Translations, locale); err != nil {
result.Errors = append(result.Errors, fmt.Sprintf("batch %d: %v", i/batchSize+1, err))
return jobOutput{result: result}
}
@@ -560,7 +567,7 @@ func (r *Result) addState(state entryState) {
}
}
-func validateBatch(entries []llm.Entry, translations map[string]string) error {
+func validateBatch(entries []llm.Entry, translations map[string]string, targetLocale string) error {
expected := make(map[string]struct{}, len(entries))
var missing []string
for _, entry := range entries {
@@ -577,7 +584,7 @@ func validateBatch(entries []llm.Entry, translations map[string]string) error {
}
if len(missing) == 0 && len(extra) == 0 {
for _, entry := range entries {
- if err := validateTranslationValue(entry.Key, entry.Value, translations[entry.Key]); err != nil {
+ if err := validateTranslationValue(entry.Key, entry.Value, translations[entry.Key], targetLocale); err != nil {
return err
}
}
diff --git a/internal/translate/translate_test.go b/internal/translate/translate_test.go
index 0b02a65..32bd5cf 100644
--- a/internal/translate/translate_test.go
+++ b/internal/translate/translate_test.go
@@ -6,9 +6,13 @@ import (
"os"
"path/filepath"
"testing"
+ "time"
"github.com/Tom-R-Main/Internationalizer/internal/config"
"github.com/Tom-R-Main/Internationalizer/internal/llm"
+ "github.com/Tom-R-Main/Internationalizer/internal/policy"
+ "github.com/Tom-R-Main/Internationalizer/internal/state"
+ "github.com/Tom-R-Main/Internationalizer/internal/tm"
validation "github.com/Tom-R-Main/Internationalizer/internal/validate"
)
@@ -54,6 +58,111 @@ func TestRunRejectsIncompleteProviderResponseWithoutWritingTarget(t *testing.T)
}
}
+func TestRunRejectsInvalidICUProviderResponseWithoutWritingTarget(t *testing.T) {
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One item} other {# items}}"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cfg := testConfig(dir, sourcePath)
+ provider := &fakeProvider{response: &llm.TranslateResponse{Translations: map[string]string{
+ "items": "{count, plural, one {Un article} other {# articles}",
+ }}}
+
+ results, err := Run(context.Background(), cfg, provider, Options{})
+ if err == nil {
+ t.Fatal("Run returned nil error for malformed ICU provider output")
+ }
+ if len(results) != 1 || len(results[0].Errors) == 0 {
+ t.Fatalf("Run results = %#v, want one locale error", results)
+ }
+ if _, statErr := os.Stat(filepath.Join(dir, "fr.json")); !os.IsNotExist(statErr) {
+ t.Fatalf("target was written after malformed ICU output; stat error = %v", statErr)
+ }
+}
+
+func TestRunRejectsMalformedICUSourceWithoutCallingProvider(t *testing.T) {
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One}}"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cfg := testConfig(dir, sourcePath)
+ provider := &fakeProvider{response: &llm.TranslateResponse{Translations: map[string]string{
+ "items": "{count, plural, one {Un} other {Autres}}",
+ }}}
+
+ results, err := Run(context.Background(), cfg, provider, Options{})
+ if err == nil {
+ t.Fatal("Run returned nil error for malformed ICU source")
+ }
+ if provider.calls != 0 {
+ t.Fatalf("provider called %d times for malformed ICU source", provider.calls)
+ }
+ if len(results) != 1 || len(results[0].Errors) == 0 {
+ t.Fatalf("Run results = %#v, want source error", results)
+ }
+}
+
+func TestRunDoesNotReuseStructurallyInvalidTMRecord(t *testing.T) {
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ source := "{count, plural, one {One item} other {# items}}"
+ if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One item} other {# items}}"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cfg := testConfig(dir, sourcePath)
+ resolved, err := policy.Resolve(cfg, "fr", "json", "", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ memory, err := tm.Load(cfg.TMPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := memory.Add(tm.Record{
+ Bundle: "default",
+ Key: "items",
+ Source: source,
+ Target: "{count, plural, one {Un article} other {# articles}",
+ Locale: "fr",
+ Hash: state.SourceHash("json", source),
+ PolicyHash: resolved.Hash,
+ Timestamp: time.Now(),
+ }); err != nil {
+ t.Fatal(err)
+ }
+ provider := &fakeProvider{response: &llm.TranslateResponse{Translations: map[string]string{
+ "items": "{count, plural, one {Un article} other {# articles}}",
+ }}}
+
+ results, err := Run(context.Background(), cfg, provider, Options{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].KeysCached != 0 || results[0].KeysTranslated != 1 || provider.calls != 1 {
+ t.Fatalf("Run results = %#v, provider calls = %d; want invalid TM bypass and fresh translation", results, provider.calls)
+ }
+}
+
+func TestRunSelectsCanonicalEquivalentConfiguredLocale(t *testing.T) {
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ if err := os.WriteFile(sourcePath, []byte(`{"save":"Save"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ cfg := testConfig(dir, sourcePath)
+ cfg.TargetLocales = []string{"pt-br"}
+
+ results, err := Run(context.Background(), cfg, nil, Options{DryRun: true, Locales: []string{"pt-BR"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].Locale != "pt-br" || results[0].TargetPath != filepath.Join(dir, "pt-br.json") {
+ t.Fatalf("Run results = %#v, want configured locale spelling and path", results)
+ }
+}
+
func TestRunProducesTargetLocalePluralFormsThatPassStrictValidation(t *testing.T) {
dir := t.TempDir()
sourcePath := filepath.Join(dir, "en.json")
diff --git a/internal/validate/findings.go b/internal/validate/findings.go
index 1b5d03b..c446101 100644
--- a/internal/validate/findings.go
+++ b/internal/validate/findings.go
@@ -13,6 +13,9 @@ const (
CodeProtectedStructureMismatch FindingCode = "protected_structure_mismatch"
CodeGlossaryViolation FindingCode = "glossary_violation"
CodePluralFormMissing FindingCode = "plural_form_missing"
+ CodeICUMessageSyntax FindingCode = "icu_message_syntax"
+ CodeICUArgumentMismatch FindingCode = "icu_argument_mismatch"
+ CodeICUSelectorMismatch FindingCode = "icu_selector_mismatch"
CodeUntracked FindingCode = "untracked"
CodeSourceStale FindingCode = "source_stale"
CodePolicyStale FindingCode = "policy_stale"
diff --git a/internal/validate/plurals.go b/internal/validate/plurals.go
index 4bb9b45..51d56ba 100644
--- a/internal/validate/plurals.go
+++ b/internal/validate/plurals.go
@@ -1,66 +1,13 @@
package validate
-import "strings"
+import (
+ "strings"
-// CLDRPluralForms maps locale codes to their required CLDR plural categories.
-// Source: Unicode CLDR plural rules for the most common languages.
-var CLDRPluralForms = map[string][]string{
- // Germanic
- "en": {"one", "other"},
- "en-US": {"one", "other"},
- "en-GB": {"one", "other"},
- "de": {"one", "other"},
- "nl": {"one", "other"},
- "sv": {"one", "other"},
- "da": {"one", "other"},
- "nb": {"one", "other"},
- "fi": {"one", "other"},
-
- // Romance
- "fr": {"one", "many", "other"},
- "fr-CA": {"one", "many", "other"},
- "es": {"one", "many", "other"},
- "es-MX": {"one", "many", "other"},
- "pt": {"one", "many", "other"},
- "pt-BR": {"one", "many", "other"},
- "it": {"one", "many", "other"},
- "ro": {"one", "few", "other"},
-
- // Slavic
- "ru": {"one", "few", "many", "other"},
- "uk": {"one", "few", "many", "other"},
- "pl": {"one", "few", "many", "other"},
- "cs": {"one", "few", "many", "other"},
-
- // Semitic
- "ar": {"zero", "one", "two", "few", "many", "other"},
- "he": {"one", "two", "other"},
-
- // Indic
- "hi": {"one", "other"},
- "bn": {"one", "other"},
- "pa": {"one", "other"},
- "te": {"one", "other"},
-
- // CJK (no plural forms — only "other")
- "ja": {"other"},
- "ko": {"other"},
- "zh": {"other"},
- "zh-CN": {"other"},
- "zh-TW": {"other"},
- "yue": {"other"},
-
- // Other
- "tr": {"one", "other"},
- "id": {"other"},
- "vi": {"other"},
- "th": {"other"},
- "el": {"one", "other"},
- "hu": {"one", "other"},
-}
+ localeid "github.com/Tom-R-Main/Internationalizer/internal/locale"
+)
// PluralFormsFor returns the CLDR plural categories for a locale.
-// Falls back to ["one", "other"] if the locale is not in the table.
+// It retains the historical ["one", "other"] fallback for an invalid tag.
func PluralFormsFor(locale string) []string {
if forms, ok := KnownPluralFormsFor(locale); ok {
return forms
@@ -68,22 +15,11 @@ func PluralFormsFor(locale string) []string {
return []string{"one", "other"}
}
-// KnownPluralFormsFor returns configured CLDR categories without guessing for
-// unknown locales. Strict validation uses this form to avoid false failures.
+// KnownPluralFormsFor returns CLDR-backed cardinal categories for a valid
+// locale. Invalid or unrecognized tags are not guessed.
func KnownPluralFormsFor(locale string) ([]string, bool) {
- if forms, ok := CLDRPluralForms[locale]; ok {
- return forms, true
- }
- for configuredLocale, forms := range CLDRPluralForms {
- if strings.EqualFold(configuredLocale, locale) {
- return forms, true
- }
- }
- if separator := strings.IndexAny(locale, "-_"); separator > 0 {
- forms, ok := CLDRPluralForms[strings.ToLower(locale[:separator])]
- return forms, ok
- }
- return nil, false
+ forms, err := localeid.CardinalCategories(locale)
+ return forms, err == nil
}
// ExpandI18nextV4Source returns the source key set plus any target-only plural
diff --git a/internal/validate/protected.go b/internal/validate/protected.go
index 2feffd9..27a3c5c 100644
--- a/internal/validate/protected.go
+++ b/internal/validate/protected.go
@@ -4,6 +4,8 @@ import (
"fmt"
"regexp"
"strings"
+
+ "github.com/Tom-R-Main/Internationalizer/internal/message"
)
var (
@@ -14,24 +16,66 @@ var (
// ProtectedFindings compares source and target structures that translations
// must preserve exactly. Multiple damaged structures may yield multiple
// findings with the same stable code and distinct messages.
-func ProtectedFindings(key, source, target string) []Finding {
+func ProtectedFindings(key, source, target, targetLocale string) []Finding {
var findings []Finding
- if mismatch := InterpolationMismatch(key, source, target); mismatch != nil {
- findings = append(findings, protectedFinding(key, "interpolation variables", mismatch.SourceVars, mismatch.TargetVars))
+ icu := usesICUValidation(source, target)
+ if !icu {
+ if mismatch := InterpolationMismatch(key, source, target); mismatch != nil {
+ findings = append(findings, protectedFinding(key, "interpolation variables", mismatch.SourceVars, mismatch.TargetVars))
+ }
}
- if sourceTags, targetTags := htmlTagRe.FindAllString(source, -1), htmlTagRe.FindAllString(target, -1); !equalStrings(sourceTags, targetTags) {
- findings = append(findings, protectedFinding(key, "HTML structure", sourceTags, targetTags))
+ checks := []struct {
+ name string
+ extract func(string) []string
+ }{
+ {"HTML structure", extractHTMLTags},
+ {"fenced code", extractFencedCode},
+ {"inline code", extractInlineCode},
+ {"markdown link destinations", extractLinkDestinations},
+ }
+ tokenizers := make([]func(string) []string, len(checks))
+ for index, check := range checks {
+ tokenizers[index] = check.extract
+ }
+ preservedKinds := make([]bool, len(checks))
+ if icu {
+ var err error
+ preservedKinds, err = message.PreservedTextTokenKinds(source, target, targetLocale, tokenizers)
+ if err != nil {
+ icu = false
+ }
}
- if sourceBlocks, targetBlocks := extractFencedCode(source), extractFencedCode(target); !equalStrings(sourceBlocks, targetBlocks) {
- findings = append(findings, protectedFinding(key, "fenced code", sourceBlocks, targetBlocks))
+ for index, check := range checks {
+ sourceTokens := check.extract(source)
+ targetTokens := check.extract(target)
+ preserved := equalStrings(sourceTokens, targetTokens)
+ if icu {
+ preserved = preservedKinds[index]
+ }
+ if !preserved {
+ findings = append(findings, protectedFinding(key, check.name, sourceTokens, targetTokens))
+ }
}
- if sourceCode, targetCode := inlineCodeRe.FindAllString(source, -1), inlineCodeRe.FindAllString(target, -1); !equalStrings(sourceCode, targetCode) {
- findings = append(findings, protectedFinding(key, "inline code", sourceCode, targetCode))
+ return findings
+}
+
+func extractHTMLTags(input string) []string {
+ return htmlTagRe.FindAllString(input, -1)
+}
+
+func extractInlineCode(input string) []string {
+ return inlineCodeRe.FindAllString(input, -1)
+}
+
+func usesICUValidation(source, target string) bool {
+ if !message.LooksLike(source) || !message.LooksLike(target) {
+ return false
}
- if sourceLinks, targetLinks := extractLinkDestinations(source), extractLinkDestinations(target); !equalStrings(sourceLinks, targetLinks) {
- findings = append(findings, protectedFinding(key, "markdown link destinations", sourceLinks, targetLinks))
+ if _, err := message.Parse(source); err != nil {
+ return false
}
- return findings
+ _, err := message.Parse(target)
+ return err == nil
}
func protectedFinding(key, structure string, expected, actual []string) Finding {
diff --git a/internal/validate/strict_test.go b/internal/validate/strict_test.go
index cec7633..6a344f1 100644
--- a/internal/validate/strict_test.go
+++ b/internal/validate/strict_test.go
@@ -221,14 +221,14 @@ func TestTargetOnlyPluralFormsReceiveContentAndStateValidation(t *testing.T) {
}
}
-func TestUnknownLocaleDoesNotGuessPluralRequirements(t *testing.T) {
+func TestPrivateUseLocaleUsesRootPluralRequirements(t *testing.T) {
cfg := validationConfig(t,
map[string]string{"items_one": "{{count}} item", "items_other": "{{count}} items"},
map[string]string{"items_one": "{{count}} x", "items_other": "{{count}} xs"},
)
oldTarget := targetPath(cfg)
- cfg.TargetLocales = []string{"xx-ZZ"}
- newTarget := filepath.Join(filepath.Dir(oldTarget), "xx-ZZ.json")
+ cfg.TargetLocales = []string{"qaa-ZZ"}
+ newTarget := filepath.Join(filepath.Dir(oldTarget), "qaa-ZZ.json")
if err := os.Rename(oldTarget, newTarget); err != nil {
t.Fatal(err)
}
@@ -239,7 +239,7 @@ func TestUnknownLocaleDoesNotGuessPluralRequirements(t *testing.T) {
t.Fatal(err)
}
if findingByCode(reports[0], CodePluralFormMissing).Code != "" {
- t.Fatalf("unknown locale received guessed plural finding: %#v", reports[0])
+ t.Fatalf("private-use locale received an unsupported plural finding: %#v", reports[0])
}
writeJSON(t, newTarget, map[string]string{"items_other": "{{count}} xs"})
@@ -247,7 +247,9 @@ func TestUnknownLocaleDoesNotGuessPluralRequirements(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- assertFindingCodes(t, reports[0], CodeMissingKey)
+ if HasFailures(reports) {
+ t.Fatalf("root plural fallback rejected an other-only target: %#v", reports[0])
+ }
}
func TestI18nextPluralCoverageUsesTargetLocaleRequirements(t *testing.T) {
@@ -336,7 +338,7 @@ func TestProtectedFindingsUseStableCodeForEveryStructure(t *testing.T) {
}
for name, values := range tests {
t.Run(name, func(t *testing.T) {
- findings := ProtectedFindings("key", values[0], values[1])
+ findings := ProtectedFindings("key", values[0], values[1], "fr")
if len(findings) == 0 || findings[0].Code != CodeProtectedStructureMismatch {
t.Fatalf("findings = %#v", findings)
}
@@ -344,6 +346,50 @@ func TestProtectedFindingsUseStableCodeForEveryStructure(t *testing.T) {
}
}
+func TestProtectedFindingsCompareICUStructuresPerLocaleBranch(t *testing.T) {
+ tests := map[string]string{
+ "html": "Save",
+ "inline code": "`go test`",
+ "fenced code": "```go\ngo test\n```\n",
+ "link": "[Guide](https://example.com/guide)",
+ }
+ for name, protected := range tests {
+ t.Run(name, func(t *testing.T) {
+ source := "{n, plural, one {" + protected + "} other {" + protected + "}}"
+ target := "{n, plural, one {" + protected + "} few {" + protected + "} many {" + protected + "} other {" + protected + "}}"
+ if findings := ProtectedFindings("key", source, target, "ru"); len(findings) != 0 {
+ t.Fatalf("valid target-only ICU branches produced findings: %#v", findings)
+ }
+
+ damaged := "{n, plural, one {" + protected + "} few {damaged} many {" + protected + "} other {" + protected + "}}"
+ if findings := ProtectedFindings("key", source, damaged, "ru"); len(findings) == 0 {
+ t.Fatal("damaged target-only ICU branch was accepted")
+ }
+ })
+ }
+}
+
+func TestProtectedFindingsCarryContextIntoNestedICUBranches(t *testing.T) {
+ argument := "{os, select, win {dir} other {ls}}"
+ damagedArgument := "{os, select, win {répertoire} other {ls}}"
+ tests := map[string][2]string{
+ "html attribute": {`Run`, `Exécuter`},
+ "inline code": {"`" + argument + "`", "`" + damagedArgument + "`"},
+ "fenced code": {"```sh\n" + argument + "\n```\n", "```sh\n" + damagedArgument + "\n```\n"},
+ "link": {"[Run](" + argument + ")", "[Exécuter](" + damagedArgument + ")"},
+ }
+ for name, values := range tests {
+ t.Run(name, func(t *testing.T) {
+ if findings := ProtectedFindings("key", values[0], values[0], "fr"); len(findings) != 0 {
+ t.Fatalf("unchanged nested ICU protection produced findings: %#v", findings)
+ }
+ if findings := ProtectedFindings("key", values[0], values[1], "fr"); len(findings) == 0 {
+ t.Fatal("damaged nested ICU protected content was accepted")
+ }
+ })
+ }
+}
+
func TestEvaluationCorpusSchemaAndIDsAreStable(t *testing.T) {
data, err := os.ReadFile(filepath.Join("..", "..", "test", "evaluation", "v1", "cases.json"))
if err != nil {
diff --git a/internal/validate/validate.go b/internal/validate/validate.go
index ccdc35f..54714ab 100644
--- a/internal/validate/validate.go
+++ b/internal/validate/validate.go
@@ -11,6 +11,7 @@ import (
"github.com/Tom-R-Main/Internationalizer/internal/config"
"github.com/Tom-R-Main/Internationalizer/internal/formats"
"github.com/Tom-R-Main/Internationalizer/internal/glossary"
+ "github.com/Tom-R-Main/Internationalizer/internal/message"
"github.com/Tom-R-Main/Internationalizer/internal/policy"
"github.com/Tom-R-Main/Internationalizer/internal/state"
"github.com/Tom-R-Main/Internationalizer/internal/styleguide"
@@ -139,6 +140,13 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s
if (opts.Strict || opts.RequireState) && pluralStyle == "i18next-v4" {
validationKeys, requiredPluralKeys, optionalPluralKeys = ExpandI18nextV4Source(sourceKeys, sourceLocale, locale)
}
+ sourceICUValid := make(map[string]bool, len(validationKeys))
+ for _, key := range allKeys(validationKeys) {
+ sourceValue := validationKeys[key]
+ findings := ICUSourceFindings(key, sourceValue, sourceLocale)
+ sourceICUValid[key] = len(findings) == 0
+ report.Findings = append(report.Findings, findings...)
+ }
targetData, err := os.ReadFile(targetPath)
if err != nil {
@@ -149,6 +157,7 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s
report.Missing = append(report.Missing, key)
report.Findings = append(report.Findings, missingFinding(key, requiredPluralKeys))
}
+ sortFindings(report.Findings)
return report
}
@@ -156,6 +165,7 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s
if err != nil {
report.Missing = allKeys(validationKeys)
report.Errors = append(report.Errors, fmt.Sprintf("parsing target: %v", err))
+ sortFindings(report.Findings)
return report
}
@@ -194,11 +204,16 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s
report.Findings = append(report.Findings, Finding{Code: CodeSourceIdentical, Severity: SeverityError, Key: key, Message: "target is identical to the source without an exact glossary exemption"})
}
- if mismatch := InterpolationMismatch(key, sourceValue, targetValue); mismatch != nil {
- report.Mismatches = append(report.Mismatches, *mismatch)
+ if !usesICUValidation(sourceValue, targetValue) {
+ if mismatch := InterpolationMismatch(key, sourceValue, targetValue); mismatch != nil {
+ report.Mismatches = append(report.Mismatches, *mismatch)
+ }
+ }
+ if sourceICUValid[key] {
+ report.Findings = append(report.Findings, ICUFindings(key, sourceValue, targetValue, locale)...)
}
if opts.Strict {
- report.Findings = append(report.Findings, ProtectedFindings(key, sourceValue, targetValue)...)
+ report.Findings = append(report.Findings, ProtectedFindings(key, sourceValue, targetValue, locale)...)
report.Findings = append(report.Findings, glossaryFindings(key, sourceValue, targetValue, terms)...)
}
if opts.RequireState {
@@ -235,6 +250,38 @@ func validateLocale(bundle, sourceLocale, locale string, sourceKeys map[string]s
return report
}
+// ICUFindings compares ICU MessageFormat structure independently of linguistic
+// strictness. Message syntax and branch identity are runtime correctness.
+func ICUFindings(key, source, target, targetLocale string) []Finding {
+ return messageFindings(key, message.Compare(source, target, targetLocale))
+}
+
+// ICUSourceFindings validates source syntax and source-locale plural selectors
+// even when a target file does not exist yet.
+func ICUSourceFindings(key, source, sourceLocale string) []Finding {
+ return messageFindings(key, message.Compare(source, source, sourceLocale))
+}
+
+func messageFindings(key string, issues []message.Issue) []Finding {
+ findings := make([]Finding, 0, len(issues))
+ for _, issue := range issues {
+ code := CodeICUArgumentMismatch
+ switch issue.Code {
+ case message.CodeSyntax:
+ code = CodeICUMessageSyntax
+ case message.CodeSelectorMismatch, message.CodeInvalidPluralCategory:
+ code = CodeICUSelectorMismatch
+ }
+ findings = append(findings, Finding{
+ Code: code,
+ Severity: SeverityError,
+ Key: key,
+ Message: issue.Message,
+ })
+ }
+ return findings
+}
+
func missingFinding(key string, requiredPluralKeys map[string]struct{}) Finding {
if _, required := requiredPluralKeys[key]; required {
return Finding{Code: CodePluralFormMissing, Severity: SeverityError, Key: key, Message: "target locale requires this plural form"}
diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go
index 2b5cbad..c61bb3e 100644
--- a/internal/validate/validate_test.go
+++ b/internal/validate/validate_test.go
@@ -3,6 +3,7 @@ package validate
import (
"os"
"path/filepath"
+ "strconv"
"strings"
"testing"
@@ -83,3 +84,128 @@ func TestFormatHumanPreservesInterpolationMismatchDetails(t *testing.T) {
t.Fatalf("FormatHuman omitted mismatch details: %q", got)
}
}
+
+func TestValidateReportsICUStructureFailures(t *testing.T) {
+ tests := []struct {
+ name string
+ locale string
+ source string
+ target string
+ code FindingCode
+ }{
+ {
+ name: "malformed target",
+ locale: "fr",
+ source: `{count, plural, one {One item} other {# items}}`,
+ target: `{count, plural, one {Un article} other {# articles}`,
+ code: CodeICUMessageSyntax,
+ },
+ {
+ name: "argument type",
+ locale: "fr",
+ source: `{count, plural, one {One item} other {# items}}`,
+ target: `{count, select, one {Un article} other {Des articles}}`,
+ code: CodeICUArgumentMismatch,
+ },
+ {
+ name: "locale category",
+ locale: "te",
+ source: `{count, plural, one {One item} other {# items}}`,
+ target: `{count, plural, one {ఒక అంశం} few {కొన్ని అంశాలు} other {# అంశాలు}}`,
+ code: CodeICUSelectorMismatch,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ report := validateMessagePair(t, test.locale, test.source, test.target)
+ if !reportHasFinding(report, test.code) || !HasFailures([]Report{report}) {
+ t.Fatalf("report = %#v, want failing finding %q", report, test.code)
+ }
+ })
+ }
+}
+
+func TestValidateAllowsLocalePluralBranchesAndExactSelectors(t *testing.T) {
+ report := validateMessagePair(t, "ru",
+ `{count, plural, one {One item} other {# items}}`,
+ `{count, plural, =0 {Нет элементов} one {Один элемент} few {# элемента} other {# элементов}}`,
+ )
+ if HasFailures([]Report{report}) {
+ t.Fatalf("valid Russian ICU message reported as failure: %#v", report)
+ }
+}
+
+func TestValidateUsesBranchAwareICUArgumentsInsteadOfFlatInterpolationCounts(t *testing.T) {
+ report := validateMessagePair(t, "ru",
+ `{count, plural, one {{name} has one item} other {{name} has # items}}`,
+ `{count, plural, one {{name} имеет один предмет} few {{name} имеет # предмета} many {{name} имеет # предметов} other {{name} имеет # предмета}}`,
+ )
+ if len(report.Mismatches) != 0 || HasFailures([]Report{report}) {
+ t.Fatalf("valid target-only ICU branches reported as failure: %#v", report)
+ }
+}
+
+func TestValidateReportsMalformedICUSourceWithoutTarget(t *testing.T) {
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ if err := os.WriteFile(sourcePath, []byte(`{"items":"{count, plural, one {One}}"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ reports, err := Validate(&config.Config{SourceLocale: "en", TargetLocales: []string{"fr"}, SourcePath: sourcePath})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(reports) != 1 || !reportHasFinding(reports[0], CodeICUMessageSyntax) {
+ t.Fatalf("reports = %#v, want source syntax finding", reports)
+ }
+}
+
+func TestValidateSortsFindingsWhenTargetIsMissing(t *testing.T) {
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ if err := os.WriteFile(sourcePath, []byte(`{"z":"{n, plural, one {One}}","a":"{n, plural, one {One}}"}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ reports, err := Validate(&config.Config{SourceLocale: "en", TargetLocales: []string{"fr"}, SourcePath: sourcePath})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(reports) != 1 || len(reports[0].Findings) != 4 {
+ t.Fatalf("reports = %#v, want four findings", reports)
+ }
+ got := reports[0].Findings
+ if got[0].Key != "a" || got[1].Key != "a" || got[2].Key != "z" || got[3].Key != "z" {
+ t.Fatalf("finding keys = [%q %q %q %q], want deterministic key order", got[0].Key, got[1].Key, got[2].Key, got[3].Key)
+ }
+}
+
+func validateMessagePair(t *testing.T, locale, source, target string) Report {
+ t.Helper()
+ dir := t.TempDir()
+ sourcePath := filepath.Join(dir, "en.json")
+ targetPath := filepath.Join(dir, locale+".json")
+ if err := os.WriteFile(sourcePath, []byte(`{"items":`+strconv.Quote(source)+`}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(targetPath, []byte(`{"items":`+strconv.Quote(target)+`}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ reports, err := Validate(&config.Config{SourceLocale: "en", TargetLocales: []string{locale}, SourcePath: sourcePath})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(reports) != 1 {
+ t.Fatalf("Validate returned %d reports, want 1", len(reports))
+ }
+ return reports[0]
+}
+
+func reportHasFinding(report Report, code FindingCode) bool {
+ for _, finding := range report.Findings {
+ if finding.Code == code {
+ return true
+ }
+ }
+ return false
+}