From 5eeed4574a416e6b0b4cb9ab2dfdbb10b84c8455 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 25 Oct 2026 19:19:14 +0200 Subject: [PATCH 1/3] feat: add JSONC support to the library, CLI, validator and REST API JSONC adds `//` and `/* */` comments and trailing commas to JSON. `.jsonc` files with comments, such as the `// Invalid:` examples written by the gts-spec example generator, were silently skipped because they were parsed as strict JSON. - Add `gts::jsonc`: `strip` blanks comments, trailing commas and a leading UTF-8 BOM in place, keeping byte offsets and line breaks so `serde_json` errors point at the original line and column. No new dependencies. - Parse `*.jsonc` as JSONC in `GtsFileReader`, `gts validate-json` (which now also discovers `.jsonc` files) and `gts-validator`; `*.json` stays strict. - Accept `--config .jsonc` and a default `gts.config.jsonc`; `gts.config.json` wins if both exist. Unreadable or malformed config files are now reported with a warning instead of being ignored silently. - REST server: accept `Content-Type: application/jsonc` bodies on every endpoint. A middleware strips them and forwards `application/json`, so the handlers, the effective `DefaultBodyLimit` and the 400/413/422 rejections are the same as for JSON. `application/json` stays strict. Signed-off-by: Artifizer --- README.md | 10 +- gts-cli/src/cli.rs | 4 +- gts-cli/src/json_validation.rs | 80 +++++++- gts-cli/src/server.rs | 52 ++++- gts-cli/tests/server_tests.rs | 147 +++++++++++++ gts-validator/README.md | 2 +- gts-validator/src/format/mod.rs | 2 +- gts-validator/src/lib.rs | 7 +- gts-validator/src/main.rs | 4 +- gts-validator/src/strategy/fs.rs | 3 +- gts-validator/src/strategy/mod.rs | 2 + gts-validator/tests/validate_fs_tests.rs | 48 +++++ gts/src/files_reader.rs | 36 +++- gts/src/jsonc.rs | 249 +++++++++++++++++++++++ gts/src/lib.rs | 1 + gts/src/ops.rs | 106 ++++++++-- 16 files changed, 708 insertions(+), 45 deletions(-) create mode 100644 gts/src/jsonc.rs diff --git a/README.md b/README.md index 5da236df..14c99090 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Other GTS spec [Reference Implementation](https://github.com/globaltypesystem/gt - [x] **x-gts-type-ref / x-gts-instance-ref** - kind-specific GTS entity references in schemas (spec §9.6.1) - [x] **x-gts-pattern-ref** - pattern-valued string fields: GTS identifier pattern syntax and coverage by an operand, independent of the registry (spec §9.6.2, §10) - [x] **YAML support** - to support YAML files (*.yml, *.yaml) as input files +- [x] **JSONC support** - to support JSONC files (*.jsonc) as input files - [ ] **TypeSpec support** - add [typespec.io](https://typespec.io/) files (*.tsp) support - [x] **UUID for instances** - to support UUID as ID in JSON instances (anonymous instances) @@ -164,7 +165,7 @@ gts validate-id --gts-id "invalid-id" Extract GTS identifiers from JSON objects. This happens automatically when loading files. ```bash -# List all entities (extracts IDs from all JSON/YAML files) +# List all entities (extracts IDs from all JSON/JSONC/YAML files) gts --path ../gts-spec/examples list --limit 10 ``` @@ -919,6 +920,11 @@ curl -X POST http://localhost:8000/entities \ -H "Content-Type: application/json" \ -d '{"gtsId": "gts.x.core.events.event.v1.0", "data": "..."}' +# Add entity from a JSONC body (comments and trailing commas allowed) +curl -X POST http://localhost:8000/entities \ + -H "Content-Type: application/jsonc" \ + --data-binary @event.jsonc + # Validate type schema (OP#12 - schema-vs-schema chain validation) curl -X POST http://localhost:8000/validate-type-schema \ -H "Content-Type: application/json" \ @@ -927,7 +933,7 @@ curl -X POST http://localhost:8000/validate-type-schema \ ## Configuration -Create a `gts.config.json` file to customize entity ID field detection: +Create a `gts.config.json` (or `gts.config.jsonc`, which may contain comments) file to customize entity ID field detection: ```json { diff --git a/gts-cli/src/cli.rs b/gts-cli/src/cli.rs index b6e87854..cfc73caf 100644 --- a/gts-cli/src/cli.rs +++ b/gts-cli/src/cli.rs @@ -52,9 +52,9 @@ pub enum Commands { #[arg(long, default_value = "major")] scope: String, }, - /// Validate all JSON documents in a file or directory + /// Validate all JSON/JSONC documents in a file or directory ValidateJson { - /// JSON file or directory to scan + /// JSON/JSONC file or directory to scan #[arg(long)] path: Option, }, diff --git a/gts-cli/src/json_validation.rs b/gts-cli/src/json_validation.rs index 6f325bd5..4ca26577 100644 --- a/gts-cli/src/json_validation.rs +++ b/gts-cli/src/json_validation.rs @@ -1,5 +1,6 @@ use gts::entities::{GtsConfig, GtsEntity, GtsFile}; use gts::gts::{GTS_ID_PREFIX, GtsId}; +use gts::jsonc; use gts::store::GtsStore; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -84,13 +85,10 @@ impl GtsJsonValidator { .unwrap_or_else(|_| self.path.clone()); if resolved.is_file() { - if resolved - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) - { + if Self::is_json_family(&resolved) { return vec![resolved]; } - self.add_issue_path(&resolved, "discovery", "Expected a .json file"); + self.add_issue_path(&resolved, "discovery", "Expected a .json or .jsonc file"); return vec![]; } @@ -120,11 +118,7 @@ impl GtsJsonValidator { continue; } - if path.is_file() - && path - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) - { + if path.is_file() && Self::is_json_family(path) { let rp = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); let key = rp.to_string_lossy().to_string(); if seen.insert(key) { @@ -137,6 +131,13 @@ impl GtsJsonValidator { files } + fn is_json_family(path: &Path) -> bool { + jsonc::is_jsonc_path(path) + || path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) + } + fn read_file(&mut self, file_path: &Path) { self.files += 1; let content_str = match std::fs::read_to_string(file_path) { @@ -147,7 +148,7 @@ impl GtsJsonValidator { } }; - let content: Value = match serde_json::from_str(&content_str) { + let content: Value = match jsonc::from_str_for_path(file_path, &content_str) { Ok(v) => v, Err(e) => { self.add_issue_path(file_path, "json", &e.to_string()); @@ -395,3 +396,60 @@ impl GtsJsonValidator { }); } } + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + const SCHEMA_JSONC: &str = r#"// Thing type +{ + "$id": "gts://gts.x.test.jsonc.thing.v1~", // type id + "$schema": "http://json-schema.org/draft-07/schema#", + /* block comment */ + "type": "object", + "properties": { + "id": { "type": "string" }, + }, +} +"#; + + fn validate(dir: &TempDir) -> GtsJsonValidationResult { + GtsJsonValidator::new(&dir.path().to_string_lossy(), GtsConfig::default()).validate() + } + + #[test] + fn validates_jsonc_documents() { + let dir = TempDir::new().unwrap(); + fs::write(dir.path().join("thing.schema.jsonc"), SCHEMA_JSONC).unwrap(); + fs::write( + dir.path().join("one.jsonc"), + "{\n // instance\n \"id\": \"gts.x.test.jsonc.thing.v1~x.test.jsonc.one.v1\",\n}\n", + ) + .unwrap(); + + let result = validate(&dir); + assert!(result.ok, "{:?}", result.issues); + assert_eq!((result.files, result.schemas, result.instances), (2, 1, 1)); + + let single = GtsJsonValidator::new( + &dir.path().join("thing.schema.jsonc").to_string_lossy(), + GtsConfig::default(), + ) + .validate(); + assert!(single.ok, "{:?}", single.issues); + assert_eq!(single.schemas, 1); + } + + #[test] + fn comments_in_json_files_are_reported() { + let dir = TempDir::new().unwrap(); + fs::write(dir.path().join("thing.schema.json"), SCHEMA_JSONC).unwrap(); + + let result = validate(&dir); + assert!(!result.ok); + assert_eq!(result.issues.len(), 1); + assert_eq!(result.issues[0].stage, "json"); + } +} diff --git a/gts-cli/src/server.rs b/gts-cli/src/server.rs index e790d29f..0cd187a3 100644 --- a/gts-cli/src/server.rs +++ b/gts-cli/src/server.rs @@ -1,16 +1,19 @@ use axum::{ Json, Router, - extract::{Path, Query, State}, - http::StatusCode, - middleware, - response::IntoResponse, + body::{Body, Bytes}, + extract::{FromRequest, Path, Query, Request, State}, + http::{HeaderMap, HeaderValue, StatusCode, header}, + middleware::{self, Next}, + response::{IntoResponse, Response}, routing::{get, post}, }; use gts::GtsOps; use gts::GtsRefValidation; +use gts::jsonc::{self, JSONC_MEDIA_TYPE}; use gts::ops::AddEntityRejection; use serde::Deserialize; use serde_json::{Value, json}; +use std::borrow::Cow; use std::sync::{Arc, Mutex}; use crate::logging::LoggingMiddleware; @@ -83,7 +86,8 @@ impl GtsHttpServer { .route("/cast", post(cast)) .route("/query", get(query)) .route("/attr", get(attr)) - .with_state(state); + .with_state(state) + .layer(middleware::from_fn(accept_jsonc_body)); // Add custom logging middleware if verbose >= 1 if verbose >= 1 { @@ -181,6 +185,44 @@ impl GtsRefValidationQuery { } } +fn is_jsonc_request(headers: &HeaderMap) -> bool { + headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .is_some_and(|mime| mime.trim().eq_ignore_ascii_case(JSONC_MEDIA_TYPE)) +} + +/// Accept `application/jsonc` bodies on every endpoint: comments and trailing +/// commas are blanked out and the request is passed on as `application/json`, +/// so the `Json` extractors and their rejections apply unchanged. +/// +/// The body is buffered with axum's `Bytes` extractor, so the effective +/// `DefaultBodyLimit` (axum's default unless a `DefaultBodyLimit` layer is +/// applied outside this middleware) and its 413/400 rejections are the same as +/// for `application/json`. +async fn accept_jsonc_body(request: Request, next: Next) -> Response { + if !is_jsonc_request(request.headers()) { + return next.run(request).await; + } + let (mut parts, body) = request.into_parts(); + let bytes = match Bytes::from_request(Request::from_parts(parts.clone(), body), &()).await { + Ok(bytes) => bytes, + Err(rejection) => return rejection.into_response(), + }; + // Stripping preserves the length, so `Content-Length` stays valid. + let stripped = match std::str::from_utf8(&bytes).map(jsonc::strip) { + Ok(Cow::Owned(text)) => Some(text), + _ => None, + }; + let body = stripped.map_or_else(|| Body::from(bytes), Body::from); + parts.headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + next.run(Request::from_parts(parts, body)).await +} + fn unprocessable(error: &str) -> axum::response::Response { ( StatusCode::UNPROCESSABLE_ENTITY, diff --git a/gts-cli/tests/server_tests.rs b/gts-cli/tests/server_tests.rs index 713c3156..88087a05 100644 --- a/gts-cli/tests/server_tests.rs +++ b/gts-cli/tests/server_tests.rs @@ -932,3 +932,150 @@ async fn test_gts_ref_validation_mode_gates_registration() { .await; assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); } + +const JSONC_SCHEMA: &str = r#"// Thing type +{ + "$id": "gts://gts.x.test.jsonc.thing.v1~", // type id + "$schema": "http://json-schema.org/draft-07/schema#", + /* block comment */ + "type": "object", + "properties": { + "id": { "type": "string" }, + }, +} +"#; + +#[allow(clippy::unwrap_used)] +async fn post(app: Router, uri: &str, content_type: &str, body: &str) -> (StatusCode, Vec) { + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(uri) + .header("content-type", content_type) + .body(Body::from(body.to_owned())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + (status, body.to_vec()) +} + +#[tokio::test] +async fn test_add_entity_accepts_jsonc_body() { + for verbose in [0, 2] { + let ops = create_test_ops(); + let state = AppState { + ops: Arc::new(Mutex::new(ops)), + }; + let app = GtsHttpServer::create_router(state.clone(), verbose); + + let (status, body) = post(app, "/entities", "application/jsonc", JSONC_SCHEMA).await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); + let result: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(result["ok"], true); + assert_eq!(result["id"], "gts.x.test.jsonc.thing.v1~"); + + let stored = state + .ops + .lock() + .unwrap() + .get_entity("gts.x.test.jsonc.thing.v1~"); + assert_eq!(stored.content.unwrap()["type"], "object"); + } +} + +#[tokio::test] +async fn test_jsonc_media_type_parameters_and_case() { + let app = create_test_router(create_test_ops(), 0); + let (status, body) = post( + app, + "/extract-id", + "Application/JSONC; charset=utf-8", + "{\"$id\": \"gts://gts.x.test.jsonc.thing.v1~\", /* c */ \"type\": \"object\",}", + ) + .await; + assert_eq!(status, StatusCode::OK); + let result: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(result["id"], "gts.x.test.jsonc.thing.v1~"); +} + +#[tokio::test] +async fn test_jsonc_bulk_body() { + let app = create_test_router(create_test_ops(), 0); + let body = format!("[\n // one schema\n {JSONC_SCHEMA},\n]"); + let (status, body) = post(app, "/type-schemas", "application/jsonc", &body).await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); +} + +#[tokio::test] +async fn test_comments_rejected_in_application_json_body() { + let app = create_test_router(create_test_ops(), 0); + let (status, _) = post(app, "/entities", "application/json", JSONC_SCHEMA).await; + assert_eq!(status, StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn test_malformed_jsonc_body_keeps_json_rejections() { + let app = create_test_router(create_test_ops(), 0); + let (status, _) = post(app, "/entities", "application/jsonc", "{\"a\": 1 /* open").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + // Well-formed JSONC with the wrong shape is a data error, as for JSON. + let app = create_test_router(create_test_ops(), 0); + let (status, _) = post(app, "/type-schemas", "application/jsonc", "{} // obj").await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); +} + +#[tokio::test] +async fn test_jsonc_body_limit_matches_json() { + // axum's default body limit applies to JSONC exactly as to JSON. + let big = format!("[{}0]", "0,".repeat(1_100_000)); + for content_type in ["application/json", "application/jsonc"] { + let app = create_test_router(create_test_ops(), 0); + let (status, _) = post(app, "/entities/bulk", content_type, &big).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE, "{content_type}"); + } + + // An outer `DefaultBodyLimit` is honored for JSONC too. + let app = + create_test_router(create_test_ops(), 0).layer(axum::extract::DefaultBodyLimit::max(16)); + let (status, _) = post( + app, + "/extract-id", + "application/jsonc", + "{\"a\": 1} // long comment", + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); +} + +#[tokio::test] +async fn test_jsonc_body_with_byte_order_mark() { + let app = create_test_router(create_test_ops(), 0); + let body = "\u{feff}{\"$id\": \"gts://gts.x.test.jsonc.thing.v1~\", // c\n}"; + let (status, body) = post(app, "/extract-id", "application/jsonc", body).await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); +} + +#[tokio::test] +#[allow(clippy::unwrap_used)] +async fn test_non_utf8_jsonc_body_is_bad_request() { + let app = create_test_router(create_test_ops(), 0); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/extract-id") + .header("content-type", "application/jsonc") + .body(Body::from(b"{\"a\": \"\xff\"} // c".to_vec())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} diff --git a/gts-validator/README.md b/gts-validator/README.md index 9d01e4ee..db103742 100644 --- a/gts-validator/README.md +++ b/gts-validator/README.md @@ -1,6 +1,6 @@ # gts-validator -GTS identifier validator for documentation and configuration files (.md, .json, .yaml). +GTS identifier validator for documentation and configuration files (.md, .json, .jsonc, .yaml, .yml). ## Overview diff --git a/gts-validator/src/format/mod.rs b/gts-validator/src/format/mod.rs index 48b38fe7..7a30c29c 100644 --- a/gts-validator/src/format/mod.rs +++ b/gts-validator/src/format/mod.rs @@ -2,7 +2,7 @@ //! //! Each sub-module handles a specific file format: //! - `markdown` — Markdown files with code-block state machine -//! - `json` — JSON tree-walker +//! - `json` — JSON tree-walker (also JSONC, after `gts::jsonc::strip`) //! - `yaml` — YAML scanner (delegates to JSON walker via `serde_json::Value`) pub mod json; diff --git a/gts-validator/src/lib.rs b/gts-validator/src/lib.rs index a53a1294..224a8137 100644 --- a/gts-validator/src/lib.rs +++ b/gts-validator/src/lib.rs @@ -151,7 +151,12 @@ pub fn validate_fs( heuristic, &validation_config.skip_tokens, ), - Some(ContentFormat::Json) => { + Some(content_format @ (ContentFormat::Json | ContentFormat::Jsonc)) => { + let content = if content_format == ContentFormat::Jsonc { + gts::jsonc::strip(&content) + } else { + content.as_str().into() + }; match format::json::scan_json_content( &content, file_path, diff --git a/gts-validator/src/main.rs b/gts-validator/src/main.rs index 878de5d1..d0229ce5 100644 --- a/gts-validator/src/main.rs +++ b/gts-validator/src/main.rs @@ -18,7 +18,7 @@ use gts_validator::{DiscoveryMode, FsSourceConfig, ValidationConfig, VendorPolic /// GTS Documentation Validator (DE0903) /// -/// Validates GTS identifiers in .md/.json/.yaml/.yml files. +/// Validates GTS identifiers in .md/.json/.jsonc/.yaml/.yml files. #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] #[allow(clippy::struct_excessive_bools)] @@ -53,7 +53,7 @@ struct Cli { #[arg(long, default_value = "10485760")] max_file_size: u64, - /// Scan JSON/YAML object keys for GTS identifiers (default: off) + /// Scan JSON/JSONC/YAML object keys for GTS identifiers (default: off) #[arg(long)] scan_keys: bool, diff --git a/gts-validator/src/strategy/fs.rs b/gts-validator/src/strategy/fs.rs index d3191d7e..131a9705 100644 --- a/gts-validator/src/strategy/fs.rs +++ b/gts-validator/src/strategy/fs.rs @@ -64,7 +64,7 @@ fn is_not_skip_dir(entry: &walkdir::DirEntry) -> bool { fn matches_file_pattern(path: &Path) -> bool { matches!( path.extension().and_then(|e| e.to_str()), - Some("md" | "json" | "yaml" | "yml") + Some("md" | "json" | "jsonc" | "yaml" | "yml") ) } @@ -214,6 +214,7 @@ pub fn content_format_for(path: &Path) -> Option { match path.extension().and_then(|e| e.to_str()) { Some("md") => Some(ContentFormat::Markdown), Some("json") => Some(ContentFormat::Json), + Some("jsonc") => Some(ContentFormat::Jsonc), Some("yaml" | "yml") => Some(ContentFormat::Yaml), _ => None, } diff --git a/gts-validator/src/strategy/mod.rs b/gts-validator/src/strategy/mod.rs index 677f6469..138faade 100644 --- a/gts-validator/src/strategy/mod.rs +++ b/gts-validator/src/strategy/mod.rs @@ -12,5 +12,7 @@ pub mod fs; pub enum ContentFormat { Markdown, Json, + /// JSON with `//` / `/* */` comments and trailing commas. + Jsonc, Yaml, } diff --git a/gts-validator/tests/validate_fs_tests.rs b/gts-validator/tests/validate_fs_tests.rs index 332193bf..2edec6d0 100644 --- a/gts-validator/tests/validate_fs_tests.rs +++ b/gts-validator/tests/validate_fs_tests.rs @@ -100,6 +100,54 @@ fn test_validate_fs_valid_json() { ); } +const JSONC_CONTENT: &str = "// Invalid: reason\n{\n /* type */ \"schema\": \"gts.x.core.pkg.mytype.v1~\", // c\n \"refs\": [\"gts.x.core.pkg.other.v1~\",],\n}\n"; + +#[test] +fn test_validate_fs_valid_jsonc() { + let tmp = TempDir::new().unwrap(); + fs::write(tmp.path().join("test.jsonc"), JSONC_CONTENT).unwrap(); + + let fs_config = default_fs_config(vec![tmp.path().to_path_buf()]); + let report = validate_fs(&fs_config, &default_validation_config()).unwrap(); + + assert_eq!(report.scanned_files, 1); + assert!( + report.ok, + "expected ok, got errors: {:?} {:?}", + report.validation_errors, report.scan_errors + ); +} + +#[test] +fn test_validate_fs_jsonc_ids_are_validated() { + let tmp = TempDir::new().unwrap(); + fs::write( + tmp.path().join("test.jsonc"), + "{\n // comment\n \"schema\": \"gts.invalid\",\n}\n", + ) + .unwrap(); + + let fs_config = default_fs_config(vec![tmp.path().to_path_buf()]); + let report = validate_fs(&fs_config, &default_validation_config()).unwrap(); + + assert_eq!(report.scanned_files, 1); + assert!(report.scan_errors.is_empty(), "{:?}", report.scan_errors); + assert_eq!(report.validation_errors.len(), 1); + assert_eq!(report.validation_errors[0].json_path, "$.schema"); +} + +#[test] +fn test_validate_fs_comments_in_json_produce_scan_error() { + let tmp = TempDir::new().unwrap(); + fs::write(tmp.path().join("test.json"), JSONC_CONTENT).unwrap(); + + let fs_config = default_fs_config(vec![tmp.path().to_path_buf()]); + let report = validate_fs(&fs_config, &default_validation_config()).unwrap(); + + assert_eq!(report.failed_files, 1); + assert!(!report.ok); +} + #[test] fn test_validate_fs_valid_yaml() { let tmp = TempDir::new().unwrap(); diff --git a/gts/src/files_reader.rs b/gts/src/files_reader.rs index e932ddc7..a14881cb 100644 --- a/gts/src/files_reader.rs +++ b/gts/src/files_reader.rs @@ -108,8 +108,8 @@ impl GtsFileReader { serde_saphyr::from_str(&content)? } _ => { - // Default: parse as JSON - serde_json::from_str(&content)? + // Default: parse as JSON (JSONC for *.jsonc) + crate::jsonc::from_str_for_path(file_path, &content)? } }; @@ -429,6 +429,38 @@ mod tests { assert_eq!(yaml_result_yml.unwrap()["name"], "test2"); } + #[test] + fn test_load_json_file_jsonc() { + let temp_dir = TempDir::new().unwrap(); + let content = + "// Invalid: reason\n{\n /* id */ \"name\": \"test\", // c\n \"value\": 42,\n}\n"; + + let jsonc_path = temp_dir.path().join("test.jsonc"); + fs::write(&jsonc_path, content).unwrap(); + let json = GtsFileReader::load_json_file(&jsonc_path).unwrap(); + assert_eq!(json, serde_json::json!({"name": "test", "value": 42})); + + // Comments stay invalid in strict *.json files. + let json_path = temp_dir.path().join("test.json"); + fs::write(&json_path, content).unwrap(); + assert!(GtsFileReader::load_json_file(&json_path).is_err()); + } + + #[test] + fn test_iter_reads_jsonc_entities() { + let temp_dir = TempDir::new().unwrap(); + fs::write( + temp_dir.path().join("schema.schema.jsonc"), + "// a schema\n{\n \"$id\": \"gts://gts.x.test.jsonc.thing.v1~\", // id\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"type\": \"object\",\n}\n", + ) + .unwrap(); + + let paths = vec![temp_dir.path().to_string_lossy().to_string()]; + let mut reader = GtsFileReader::new(&paths, None); + let ids: Vec = reader.iter().filter_map(|e| e.effective_id()).collect(); + assert_eq!(ids, vec!["gts.x.test.jsonc.thing.v1~".to_owned()]); + } + #[test] fn test_load_json_file_invalid_json() { let temp_dir = TempDir::new().unwrap(); diff --git a/gts/src/jsonc.rs b/gts/src/jsonc.rs new file mode 100644 index 00000000..4ffbaa88 --- /dev/null +++ b/gts/src/jsonc.rs @@ -0,0 +1,249 @@ +//! JSONC (JSON with Comments) support. +//! +//! JSONC extends JSON with `//` line comments, `/* */` block comments and +//! trailing commas before a closing `}` or `]`. [`strip`] blanks those out in +//! place: every removed byte becomes a space and line breaks are kept, so the +//! result is strict JSON with the same byte offsets as the input and +//! `serde_json` errors keep pointing at the original line and column. +//! +//! Only `*.jsonc` files (see [`is_jsonc_path`]) and request bodies sent as +//! [`JSONC_MEDIA_TYPE`] are parsed as JSONC; `*.json` stays strict JSON. + +use std::borrow::Cow; +use std::path::Path; + +use serde::de::DeserializeOwned; + +/// File extension (without the dot) of JSONC documents. +pub const JSONC_EXTENSION: &str = "jsonc"; + +/// Media type of JSONC request bodies. +pub const JSONC_MEDIA_TYPE: &str = "application/jsonc"; + +/// Whether `path` names a JSONC document, i.e. has a `.jsonc` extension (case-insensitive). +#[must_use] +pub fn is_jsonc_path(path: &Path) -> bool { + path.extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case(JSONC_EXTENSION)) +} + +/// Turn JSONC into strict JSON by blanking comments, trailing commas and a +/// leading UTF-8 byte order mark. +/// +/// Byte offsets and line breaks are preserved. Malformed input (e.g. an +/// unterminated block comment or string) is left as is for the JSON parser to +/// reject at the right position. +#[must_use] +pub fn strip(input: &str) -> Cow<'_, str> { + let bytes = input.as_bytes(); + let mut out: Option> = None; + let mut blank = |range: std::ops::Range| { + let out = out.get_or_insert_with(|| bytes.to_vec()); + for b in &mut out[range] { + if !matches!(*b, b'\n' | b'\r') { + *b = b' '; + } + } + }; + + // Last significant byte seen outside strings and comments, and the + // position of a comma that may turn out to be trailing. + let mut prev = 0u8; + let mut pending_comma: Option = None; + let mut i = 0; + if input.starts_with('\u{feff}') { + i = '\u{feff}'.len_utf8(); + blank(0..i); + } + while i < bytes.len() { + match (bytes[i], bytes.get(i + 1)) { + (b'"', _) => { + i += 1; + while i < bytes.len() { + match bytes[i] { + b'\\' => i += 2, + b'"' => break, + _ => i += 1, + } + } + i += 1; + prev = b'"'; + pending_comma = None; + } + (b'/', Some(b'/')) => { + let end = bytes[i..] + .iter() + .position(|b| matches!(b, b'\n' | b'\r')) + .map_or(bytes.len(), |len| i + len); + blank(i..end); + i = end; + } + (b'/', Some(b'*')) => { + let Some(len) = input[i + 2..].find("*/") else { + break; + }; + let end = i + 2 + len + 2; + blank(i..end); + i = end; + } + (b' ' | b'\t' | b'\n' | b'\r', _) => i += 1, + (b, _) => { + match b { + b',' => { + pending_comma = + (!matches!(prev, 0 | b'[' | b'{' | b',' | b':')).then_some(i); + } + b'}' | b']' => { + if let Some(comma) = pending_comma.take() { + blank(comma..comma + 1); + } + } + _ => pending_comma = None, + } + prev = b; + i += 1; + } + } + } + + // Only whole characters were replaced by ASCII spaces, so the output is + // still valid UTF-8 and the lossy fallback is unreachable. + out.map_or(Cow::Borrowed(input), |out| { + Cow::Owned( + String::from_utf8(out) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()), + ) + }) +} + +/// Deserialize a JSONC document. +/// +/// # Errors +/// +/// Returns the `serde_json` error for input that is not valid JSONC or does +/// not match `T`; line and column refer to the original input. +pub fn from_str(input: &str) -> serde_json::Result { + serde_json::from_str(&strip(input)) +} + +/// Deserialize `content` read from `path`: as JSONC for `*.jsonc`, as strict JSON otherwise. +/// +/// # Errors +/// +/// Returns the `serde_json` error for content that does not parse or does not match `T`. +pub fn from_str_for_path(path: &Path, content: &str) -> serde_json::Result { + if is_jsonc_path(path) { + from_str(content) + } else { + serde_json::from_str(content) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{Value, json}; + + fn parse(input: &str) -> serde_json::Result { + from_str(input) + } + + #[test] + fn strict_json_is_borrowed_unchanged() { + let input = r#"{"a": [1, 2], "b": "x"}"#; + assert!(matches!(strip(input), Cow::Borrowed(s) if s == input)); + } + + #[test] + fn line_and_block_comments_are_removed() { + let input = "// Invalid: reason\n{\n /* block\n comment */ \"a\": 1, // trailing\n \"b\": 2\n}\n"; + assert_eq!(parse(input).unwrap(), json!({"a": 1, "b": 2})); + } + + #[test] + fn comments_with_crlf_and_eof() { + assert_eq!(parse("{\"a\": 1}\r\n// end").unwrap(), json!({"a": 1})); + assert_eq!(parse("[1 /**/, 2]").unwrap(), json!([1, 2])); + } + + #[test] + fn trailing_commas_are_removed() { + assert_eq!( + parse("{\"a\": [1, 2,], \"b\": {\"c\": 3, },}").unwrap(), + json!({"a": [1, 2], "b": {"c": 3}}) + ); + assert_eq!(parse("[1, // c\n /* d */ ]").unwrap(), json!([1])); + assert_eq!(parse("[\"s\",\n]").unwrap(), json!(["s"])); + } + + #[test] + fn comment_markers_inside_strings_are_kept() { + let input = r#"{"url": "gts://gts.x.a.b.c.v1~", "c": "/* not */", "e": "q\"//,]"}"#; + assert!(matches!(strip(input), Cow::Borrowed(_))); + assert_eq!( + parse(input).unwrap(), + json!({"url": "gts://gts.x.a.b.c.v1~", "c": "/* not */", "e": "q\"//,]"}) + ); + } + + #[test] + fn non_ascii_in_comments_and_strings() { + let input = "{\"name\": \"caf\u{e9}\" /* \u{fc}ber */, // \u{65e5}\u{672c}\n}"; + assert_eq!(parse(input).unwrap(), json!({"name": "caf\u{e9}"})); + } + + #[test] + fn leading_byte_order_mark_is_removed() { + let input = "\u{feff}// c\n{\"a\": 1,}"; + assert_eq!(strip(input).len(), input.len()); + assert_eq!(parse(input).unwrap(), json!({"a": 1})); + assert_eq!(parse("\u{feff}[1]").unwrap(), json!([1])); + // Only a leading mark is a byte order mark. + assert!(parse("[1]\u{feff}").is_err()); + } + + #[test] + fn offsets_and_lines_are_preserved() { + let input = "{\n // c\n \"a\": 1,\n}"; + let stripped = strip(input); + assert_eq!(stripped.len(), input.len()); + assert_eq!(stripped.lines().count(), input.lines().count()); + + let err = parse("// header\n{\n \"a\": nope\n}").unwrap_err(); + let strict = serde_json::from_str::("\n{\n \"a\": nope\n}").unwrap_err(); + assert_eq!(err.line(), 3); + assert_eq!((err.line(), err.column()), (strict.line(), strict.column())); + } + + #[test] + fn malformed_input_is_still_rejected() { + for input in [ + "{\"a\": 1 /* unterminated", + "[,]", + "[1,,]", + "{\"a\":,}", + "{,}", + ",", + "[1 2]", + "{'a': 1}", + "\"unterminated", + ] { + assert!(parse(input).is_err(), "{input:?} must not parse"); + } + } + + #[test] + fn path_detection_and_dispatch() { + assert!(is_jsonc_path(Path::new("a/b.jsonc"))); + assert!(is_jsonc_path(Path::new("B.JSONC"))); + assert!(!is_jsonc_path(Path::new("b.json"))); + assert!(!is_jsonc_path(Path::new("jsonc"))); + + let content = "{\"a\": 1, // c\n}"; + assert_eq!( + from_str_for_path::(Path::new("x.jsonc"), content).unwrap(), + json!({"a": 1}) + ); + assert!(from_str_for_path::(Path::new("x.json"), content).is_err()); + } +} diff --git a/gts/src/lib.rs b/gts/src/lib.rs index fd86b9b7..8c89de35 100644 --- a/gts/src/lib.rs +++ b/gts/src/lib.rs @@ -2,6 +2,7 @@ pub mod entities; pub mod files_reader; pub mod gts; mod json_schema; +pub mod jsonc; pub mod ops; pub mod path_resolver; pub mod regex_profile; diff --git a/gts/src/ops.rs b/gts/src/ops.rs index 1648ce2b..26770a7d 100644 --- a/gts/src/ops.rs +++ b/gts/src/ops.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; use std::fs; -use std::path::PathBuf; +use std::path::Path; use crate::entities::{GtsConfig, GtsEntity}; use crate::files_reader::GtsFileReader; @@ -262,7 +262,7 @@ pub struct GtsOps { impl GtsOps { #[must_use] pub fn new(path: Option>, config: Option, verbose: usize) -> Self { - let cfg = Self::load_config(config); + let cfg = Self::load_config_in(config, Path::new("")); let store = match path.as_ref() { Some(p) => { let reader = Box::new(GtsFileReader::new(p, Some(cfg.clone()))) @@ -280,28 +280,45 @@ impl GtsOps { } } - fn load_config(config_path: Option) -> GtsConfig { - // Try user-provided path - if let Some(path) = config_path - && let Ok(cfg) = Self::load_config_from_path(&PathBuf::from(path)) - { - return cfg; - } + /// Load the user-provided config, else the first existing default config + /// file in `dir`, else the defaults. Unreadable or malformed files are + /// skipped with a warning. + fn load_config_in(config_path: Option, dir: &Path) -> GtsConfig { + config_path + .and_then(|path| Self::try_load_config(Path::new(&path))) + .or_else(|| Self::default_config_path(dir).and_then(|p| Self::try_load_config(&p))) + .unwrap_or_default() + } - // Try default path (relative to current directory) + /// The first of `gts.config.json` / `gts.config.jsonc` that exists in `dir`. + fn default_config_path(dir: &Path) -> Option { #[allow(unknown_lints, gts_id_hardcoded_prefix)] - let default_path = PathBuf::from("gts.config.json"); - if let Ok(cfg) = Self::load_config_from_path(&default_path) { - return cfg; + let default_names = ["gts.config.json", "gts.config.jsonc"]; + let mut existing = default_names + .iter() + .map(|name| dir.join(name)) + .filter(|path| path.is_file()); + let path = existing.next()?; + for ignored in existing { + tracing::warn!( + "Both {} and {} exist; using {}", + path.display(), + ignored.display(), + path.display() + ); } + Some(path) + } - // Fall back to defaults - GtsConfig::default() + fn try_load_config(path: &Path) -> Option { + Self::load_config_from_path(path) + .inspect_err(|e| tracing::warn!("Ignoring GTS config {}: {e}", path.display())) + .ok() } - fn load_config_from_path(path: &PathBuf) -> Result> { + fn load_config_from_path(path: &Path) -> Result> { let content = fs::read_to_string(path)?; - let data: HashMap = serde_json::from_str(&content)?; + let data: HashMap = crate::jsonc::from_str_for_path(path, &content)?; Ok(Self::create_config_from_data(&data)) } @@ -4121,6 +4138,61 @@ mod tests { assert_eq!(config.type_id_fields, default_cfg.type_id_fields); } + #[test] + fn test_load_config_from_jsonc_path() { + let temp_dir = tempfile::TempDir::new().unwrap(); + let content = "{\n // custom id fields\n \"entity_id_fields\": [\"customId\",],\n}\n"; + + let jsonc_path = temp_dir.path().join("config.jsonc"); + fs::write(&jsonc_path, content).unwrap(); + let config = GtsOps::load_config_from_path(&jsonc_path).unwrap(); + assert_eq!(config.entity_id_fields, vec!["customId"]); + + let json_path = temp_dir.path().join("config.json"); + fs::write(&json_path, content).unwrap(); + assert!(GtsOps::load_config_from_path(&json_path).is_err()); + } + + #[test] + fn test_load_config_in_default_files() { + let dir = tempfile::TempDir::new().unwrap(); + let default_fields = GtsConfig::default().entity_id_fields; + let load = + |config: Option<&str>| GtsOps::load_config_in(config.map(str::to_owned), dir.path()); + let jsonc_path = dir.path().join("gts.config.jsonc"); + let json_path = dir.path().join("gts.config.json"); + + assert_eq!(load(None).entity_id_fields, default_fields); + + fs::write( + &jsonc_path, + "{\"entity_id_fields\": [\"fromJsonc\"], // c\n}", + ) + .unwrap(); + assert_eq!(load(None).entity_id_fields, vec!["fromJsonc"]); + + // gts.config.json takes precedence over gts.config.jsonc. + fs::write(&json_path, r#"{"entity_id_fields": ["fromJson"]}"#).unwrap(); + assert_eq!(load(None).entity_id_fields, vec!["fromJson"]); + + // A malformed preferred file falls back to the defaults, not to the other file. + fs::write(&json_path, "{ // not json").unwrap(); + assert_eq!(load(None).entity_id_fields, default_fields); + + // An explicit config wins; a broken one falls back to the default files. + let custom = dir.path().join("custom.jsonc"); + fs::write(&custom, "{\"entity_id_fields\": [\"custom\"]}").unwrap(); + assert_eq!( + load(Some(&custom.to_string_lossy())).entity_id_fields, + vec!["custom"] + ); + fs::remove_file(&json_path).unwrap(); + assert_eq!( + load(Some("/nonexistent/gts.config.json")).entity_id_fields, + vec!["fromJsonc"] + ); + } + #[test] fn test_create_config_from_data_with_invalid_types() { let mut data = HashMap::new(); From 5152bbd394090e29a2f60f17fd93842dbfbdb58d Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 25 Oct 2026 19:19:21 +0200 Subject: [PATCH 2/3] fix(gts-cli): write logs to stderr The tracing subscriber wrote to stdout, so any warning was mixed into the JSON results the CLI prints there and broke piping into tools such as `jq`. Server `-v`/`-vv` logs move to stderr as well; the request/response logging middleware already used stderr. Signed-off-by: Artifizer --- gts-cli/src/cli.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gts-cli/src/cli.rs b/gts-cli/src/cli.rs index cfc73caf..40887397 100644 --- a/gts-cli/src/cli.rs +++ b/gts-cli/src/cli.rs @@ -168,7 +168,9 @@ pub async fn run_with_cli(cli: Cli) -> Result<()> { }; // Only initialize logging if not already initialized (for testing) + // Logs go to stderr so they never corrupt the JSON results on stdout. let _ = tracing_subscriber::fmt() + .with_writer(std::io::stderr) .with_max_level(log_level) .with_target(false) .try_init(); From 10dd794b6c0ed1812306bb65f9c2256c35d3c3f6 Mon Sep 17 00:00:00 2001 From: Artifizer Date: Sun, 25 Oct 2026 19:19:21 +0200 Subject: [PATCH 3/3] chore: bump GTS crates to v0.16.3 Signed-off-by: Artifizer --- Cargo.lock | 12 ++++++------ Cargo.toml | 12 ++++++------ gts-cli/Cargo.toml | 2 +- gts-dylint/Cargo.lock | 8 ++++---- gts-dylint/Cargo.toml | 2 +- gts-id/Cargo.toml | 2 +- gts-macros-cli/Cargo.toml | 2 +- gts-macros/Cargo.toml | 2 +- gts-validator/Cargo.toml | 2 +- gts/Cargo.toml | 2 +- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76dad302..76814460 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -539,7 +539,7 @@ dependencies = [ [[package]] name = "gts" -version = "0.16.2" +version = "0.16.3" dependencies = [ "gts-id", "jsonschema", @@ -561,7 +561,7 @@ dependencies = [ [[package]] name = "gts-cli" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "axum", @@ -583,7 +583,7 @@ dependencies = [ [[package]] name = "gts-id" -version = "0.16.2" +version = "0.16.3" dependencies = [ "thiserror", "uuid", @@ -591,7 +591,7 @@ dependencies = [ [[package]] name = "gts-macros" -version = "0.16.2" +version = "0.16.3" dependencies = [ "gts", "gts-id", @@ -608,7 +608,7 @@ dependencies = [ [[package]] name = "gts-macros-cli" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "clap", @@ -623,7 +623,7 @@ dependencies = [ [[package]] name = "gts-validator" -version = "0.16.2" +version = "0.16.3" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index a72deac4..babc1f88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,12 +153,12 @@ verbose_file_reads = "deny" module_name_repetitions = "allow" [workspace.dependencies] -gts = { version = "0.16.2", path = "gts" } -gts-cli = { version = "0.16.2", path = "gts-cli" } -gts-id = { version = "0.16.2", path = "gts-id" } -gts-macros = { version = "0.16.2", path = "gts-macros" } -gts-macros-cli = { version = "0.16.2", path = "gts-macros-cli" } -gts-validator = { version = "0.16.2", path = "gts-validator" } +gts = { version = "0.16.3", path = "gts" } +gts-cli = { version = "0.16.3", path = "gts-cli" } +gts-id = { version = "0.16.3", path = "gts-id" } +gts-macros = { version = "0.16.3", path = "gts-macros" } +gts-macros-cli = { version = "0.16.3", path = "gts-macros-cli" } +gts-validator = { version = "0.16.3", path = "gts-validator" } # Core dependencies serde = { version = "1.0", features = ["derive"] } diff --git a/gts-cli/Cargo.toml b/gts-cli/Cargo.toml index 0a7148c8..73fb75a8 100644 --- a/gts-cli/Cargo.toml +++ b/gts-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts-cli" -version = "0.16.2" +version = "0.16.3" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/gts-dylint/Cargo.lock b/gts-dylint/Cargo.lock index 43cc4514..3d91fa22 100644 --- a/gts-dylint/Cargo.lock +++ b/gts-dylint/Cargo.lock @@ -645,7 +645,7 @@ dependencies = [ [[package]] name = "gts" -version = "0.16.2" +version = "0.16.3" dependencies = [ "gts-id", "jsonschema", @@ -666,7 +666,7 @@ dependencies = [ [[package]] name = "gts-dylint" -version = "0.16.2" +version = "0.16.3" dependencies = [ "dylint_linting", "dylint_testing", @@ -680,7 +680,7 @@ dependencies = [ [[package]] name = "gts-id" -version = "0.16.2" +version = "0.16.3" dependencies = [ "thiserror 2.0.18", "uuid", @@ -688,7 +688,7 @@ dependencies = [ [[package]] name = "gts-macros" -version = "0.16.2" +version = "0.16.3" dependencies = [ "gts-id", "proc-macro2", diff --git a/gts-dylint/Cargo.toml b/gts-dylint/Cargo.toml index 2795c81f..f6d2f992 100644 --- a/gts-dylint/Cargo.toml +++ b/gts-dylint/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts-dylint" -version = "0.16.2" +version = "0.16.3" edition = "2024" publish = false diff --git a/gts-id/Cargo.toml b/gts-id/Cargo.toml index 3cd01757..42758bc5 100644 --- a/gts-id/Cargo.toml +++ b/gts-id/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts-id" -version = "0.16.2" +version = "0.16.3" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/gts-macros-cli/Cargo.toml b/gts-macros-cli/Cargo.toml index 50d6900b..aa711efc 100644 --- a/gts-macros-cli/Cargo.toml +++ b/gts-macros-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts-macros-cli" -version = "0.16.2" +version = "0.16.3" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/gts-macros/Cargo.toml b/gts-macros/Cargo.toml index 1a74d55a..05bcabdc 100644 --- a/gts-macros/Cargo.toml +++ b/gts-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts-macros" -version = "0.16.2" +version = "0.16.3" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/gts-validator/Cargo.toml b/gts-validator/Cargo.toml index dbfb4949..e8b3d5a9 100644 --- a/gts-validator/Cargo.toml +++ b/gts-validator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts-validator" -version = "0.16.2" +version = "0.16.3" edition.workspace = true authors.workspace = true license.workspace = true diff --git a/gts/Cargo.toml b/gts/Cargo.toml index ae202a3e..1f5e89b6 100644 --- a/gts/Cargo.toml +++ b/gts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gts" -version = "0.16.2" +version = "0.16.3" edition.workspace = true authors.workspace = true license.workspace = true