feat: add locale and ICU correctness core

README.md

@@ -138,8 +138,9 @@ baseline.
 ### `validate`
 
 Check all locale files against their source bundles. Default validation checks
-structural coverage (the percentage of required target keys present), reports extra keys
-as warnings, and fails for missing keys or interpolation mismatches.
+structural coverage (the percentage of required target keys present), reports
+extra keys as warnings, and fails for missing keys, interpolation mismatches,
+or invalid ICU MessageFormat structure.
 
 ```bash
 internationalizer validate                     # human-readable output
@@ -170,6 +171,9 @@ Human and JSON reports use stable finding codes:
 | `protected_structure_mismatch` | Interpolation, HTML, code, or link structure changed |
 | `glossary_violation` | No approved target term or variant was found |
 | `plural_form_missing` | A configured locale plural form is absent |
+| `icu_message_syntax` | A source or target ICU message is malformed |
+| `icu_argument_mismatch` | ICU argument names, types, or formatter styles differ |
+| `icu_selector_mismatch` | Selectors differ or a plural category is invalid for the target locale |
 | `untracked` | No manifest record exists for the target |
 | `source_stale` | Source content changed after the recorded translation |
 | `policy_stale` | The generated prompt or model settings changed |
@@ -299,6 +303,18 @@ validation:
   plural_style: i18next-v4 # generate and validate target-locale plural forms
 ```
 
+Locale identifiers must be well-formed BCP 47 tags such as `fr`, `pt-BR`, or
+`sr-Latn-RS`. Canonical-equivalent target locales are rejected as duplicates,
+and locale-specific provider overrides match canonical-equivalent spelling.
+
+ICU MessageFormat values are parsed structurally. Simple arguments, `select`,
+`plural`, `selectordinal`, `number`, `date`, and `time` are supported, including
+nested messages, plural offsets, exact-number selectors, and `#`. Validation
+checks syntax, argument types and formatter styles, plural offsets, select
+branch identity, and target-locale CLDR plural categories. Provider output that
+breaks these invariants is rejected before a locale file or translation-memory
+record is written.
+
 With `i18next-v4`, recognized source plural families are expanded during
 translation to the target locale's CLDR categories. A target-only category uses
 the source family's `_other` value as its translation template. Strict
@@ -392,6 +408,8 @@ internal/
     openai.go              OpenAI / compatible backend
     gemini.go              Google Gemini via AI Studio backend
                            OpenRouter uses openai.go with custom base_url
+  locale/                  BCP 47 identity and CLDR plural categories
+  message/                 ICU MessageFormat parser and structural comparison
   styleguide/              Style guide loader
   tm/                      JSONL translation memory
   translate/               Translation orchestrator
@@ -415,6 +433,8 @@ internal/
 
 [AGPL-3.0](LICENSE)
 
+See [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for dependency notices.
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines. All contributions require DCO sign-off.

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,32 @@
+# Third-party notices
+
+Internationalizer includes software developed by third parties. Their licenses
+apply to those components; the Internationalizer source remains licensed under
+AGPL-3.0.
+
+## golang.org/x/text
+
+Copyright 2009 The Go Authors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+- Neither the name of Google LLC nor the names of its contributors may be used
+  to endorse or promote products derived from this software without specific
+  prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cmd/internationalizer/translate.go

@@ -48,8 +48,12 @@ func newTranslateCmd() *cobra.Command {
 					selectedLocales = cfg.TargetLocales
 				}
 				needsDefaultProvider := false
-				for _, locale := range selectedLocales {
-					if _, ok := cfg.LLM.LocaleOverrides[locale]; !ok {
+				for _, requested := range selectedLocales {
+					locale, ok := cfg.ConfiguredTargetLocale(requested)
+					if !ok {
+						return fmt.Errorf("locale %q is not in target_locales", requested)
+					}
+					if !cfg.HasLLMOverrideForLocale(locale) {
 						needsDefaultProvider = true
 						break
 					}
@@ -60,8 +64,12 @@ func newTranslateCmd() *cobra.Command {
 						return err
 					}
 				}
-				for _, locale := range selectedLocales {
-					if _, ok := cfg.LLM.LocaleOverrides[locale]; !ok {
+				for _, requested := range selectedLocales {
+					locale, ok := cfg.ConfiguredTargetLocale(requested)
+					if !ok {
+						return fmt.Errorf("locale %q is not in target_locales", requested)
+					}
+					if !cfg.HasLLMOverrideForLocale(locale) {
 						continue
 					}
 					localeLLM := cfg.LLMForLocale(locale)

docs/localization-core-v2.md

@@ -0,0 +1,49 @@
+# Localization Core v2 contract
+
+Internationalizer remains a local-first translation compiler for applications
+that own their runtime. This contract does not add a hosted review service,
+framework runtime, translation CDN, or source-rewriting setup command.
+
+## Locale identity
+
+- Configuration accepts well-formed BCP 47 language tags and preserves their
+  spelling in file paths.
+- Equality, provider overrides, validation, and state identity use canonical
+  BCP 47 form. Canonical-equivalent target locales are duplicates.
+- Private-use language tags remain valid. Path separators and malformed tags
+  are rejected before resolving target paths.
+
+## Message semantics
+
+- ICU messages are parsed structurally rather than treated as brace-shaped
+  interpolation strings.
+- Supported arguments are simple interpolation, `select`, `plural`,
+  `selectordinal`, `number`, `date`, and `time`.
+- Selectors may contain nested messages. Plural arguments support exact-number
+  selectors, offsets, and `#`.
+- Every `select`, `plural`, and `selectordinal` requires `other`.
+- A target must retain source argument names, argument kinds, formatter styles,
+  and plural offsets. Select branch identities are semantic and must be
+  preserved. Target plurals may add exact selectors and locale-applicable CLDR
+  categories.
+- Parsing and printing are deterministic so later source-unit hashing can use
+  canonical message structure.
+
+## Review transfer
+
+Manifest v2 will represent generated, needs-review, approved, and manual
+origins explicitly. Only an exact source-and-policy match may inherit approval.
+Fuzzy candidates, changed policy, or changed message structure require review.
+
+## Source units
+
+Structured documents and source adapters will identify a unit by adapter,
+normalized project-relative path, semantic location, and source structure.
+Formatting offsets alone are not stable identity.
+
+## Compatibility boundary
+
+Existing JSON/YAML key workflows and i18next-v4 plural suffixes remain
+supported. Canonical locale comparison may reject configurations that currently
+spell the same locale more than once. ICU structural failures are deterministic
+validation errors and block provider output from being written.

go.mod

@@ -1,9 +1,10 @@
 module github.com/Tom-R-Main/Internationalizer
 
-go 1.25
+go 1.25.0
 
 require (
 	github.com/spf13/cobra v1.10.2
+	golang.org/x/text v0.36.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -8,6 +8,8 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
 github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/config/config.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
+	"sort"
 	"strings"
 
+	localeid "github.com/Tom-R-Main/Internationalizer/internal/locale"
 	"gopkg.in/yaml.v3"
 )
 
@@ -133,7 +134,7 @@ func (c *Config) LLMForLocale(locale string) LLM {
 	global.LocaleOverrides = nil
 	applyLLMDefaults(&global)
 
-	override, ok := c.LLM.LocaleOverrides[locale]
+	override, ok := localeOverride(c.LLM.LocaleOverrides, locale)
 	if !ok {
 		return global
 	}
@@ -165,6 +166,29 @@ func (c *Config) LLMForLocale(locale string) LLM {
 	return effective
 }
 
+// ConfiguredTargetLocale returns the configured spelling of a target locale
+// that is canonically equivalent to requested.
+func (c *Config) ConfiguredTargetLocale(requested string) (string, bool) {
+	canonical, err := localeid.Canonical(requested)
+	if err != nil {
+		return "", false
+	}
+	for _, candidate := range c.TargetLocales {
+		candidateCanonical, candidateErr := localeid.Canonical(candidate)
+		if candidateErr == nil && candidateCanonical == canonical {
+			return candidate, true
+		}
+	}
+	return "", false
+}
+
+// HasLLMOverrideForLocale reports whether a canonical-equivalent locale has
+// explicit provider settings.
+func (c *Config) HasLLMOverrideForLocale(locale string) bool {
+	_, ok := localeOverride(c.LLM.LocaleOverrides, locale)
+	return ok
+}
+
 func (c *Config) Validate() error {
 	if err := c.ValidateProject(); err != nil {
 		return err
@@ -177,24 +201,38 @@ func (c *Config) ValidateProject() error {
 	if c.Validation.PluralStyle != "" && c.Validation.PluralStyle != "i18next-v4" {
 		return fmt.Errorf("unsupported validation.plural_style %q", c.Validation.PluralStyle)
 	}
+	sourceLocale := c.SourceLocale
+	if sourceLocale == "" {
+		sourceLocale = "en"
+	}
+	if _, err := localeid.Canonical(sourceLocale); err != nil {
+		return fmt.Errorf("invalid source locale %q: %w", c.SourceLocale, err)
+	}
 	if len(c.TargetLocales) == 0 {
 		return fmt.Errorf("target_locales must not be empty")
 	}
 	localeSeen := make(map[string]struct{}, len(c.TargetLocales))
 	for _, locale := range c.TargetLocales {
-		if !validLocale.MatchString(locale) {
-			return fmt.Errorf("invalid target locale %q", locale)
+		canonical, err := localeid.Canonical(locale)
+		if err != nil {
+			return fmt.Errorf("invalid target locale %q: %w", locale, err)
 		}
-		if _, ok := localeSeen[locale]; ok {
+		if _, ok := localeSeen[canonical]; ok {
 			return fmt.Errorf("duplicate target locale %q", locale)
 		}
-		localeSeen[locale] = struct{}{}
+		localeSeen[canonical] = struct{}{}
 	}
+	overrideSeen := make(map[string]struct{}, len(c.LLM.LocaleOverrides))
 	for locale := range c.LLM.LocaleOverrides {
-		if !validLocale.MatchString(locale) {
-			return fmt.Errorf("invalid llm locale override %q", locale)
+		canonical, err := localeid.Canonical(locale)
+		if err != nil {
+			return fmt.Errorf("invalid llm locale override %q: %w", locale, err)
 		}
-		if _, ok := localeSeen[locale]; !ok {
+		if _, duplicate := overrideSeen[canonical]; duplicate {
+			return fmt.Errorf("duplicate llm locale override %q", locale)
+		}
+		overrideSeen[canonical] = struct{}{}
+		if _, ok := localeSeen[canonical]; !ok {
 			return fmt.Errorf("llm locale override %q is not in target_locales", locale)
 		}
 	}
@@ -276,8 +314,6 @@ func validateAPIKeyEnv(apiKeyEnv string, checked map[string]struct{}) error {
 	return nil
 }
 
-var validLocale = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`)
-
 // EffectiveBundles returns explicit bundles, or a backward-compatible bundle
 // derived from source_path and the historical sibling target convention.
 func (c *Config) EffectiveBundles() []Bundle {
@@ -298,15 +334,36 @@ func (c *Config) EffectiveBundles() []Bundle {
 
 // TargetPath resolves a bundle target for a validated locale.
 func (b Bundle) TargetPath(locale string) (string, error) {
-	if !validLocale.MatchString(locale) {
-		return "", fmt.Errorf("invalid target locale %q", locale)
+	if _, err := localeid.Canonical(locale); err != nil {
+		return "", fmt.Errorf("invalid target locale %q: %w", locale, err)
 	}
 	if !strings.Contains(b.Target, "{locale}") {
 		return "", fmt.Errorf("bundle %q target must contain {locale}", b.ID)
 	}
 	return filepath.Clean(strings.ReplaceAll(b.Target, "{locale}", locale)), nil
 }
 
+func localeOverride(overrides map[string]LLMOverride, requested string) (LLMOverride, bool) {
+	if override, ok := overrides[requested]; ok {
+		return override, true
+	}
+	canonical, err := localeid.Canonical(requested)
+	if err != nil {
+		return LLMOverride{}, false
+	}
+	keys := make([]string, 0, len(overrides))
+	for key := range overrides {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		if candidate, candidateErr := localeid.Canonical(key); candidateErr == nil && candidate == canonical {
+			return overrides[key], true
+		}
+	}
+	return LLMOverride{}, false
+}
+
 // APIKey returns the resolved API key from the environment.
 func (c *Config) APIKey() string {
 	return os.Getenv(c.LLM.APIKeyEnv)