diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c1dbb6264..aadb81db3 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -4,3 +4,6 @@ # docs/tasks/0060: braces around every single-statement body (clang-format InsertBraces) 33b30998156056c4de59dcb0ce598390602ab253 + +# docs/tasks/0076: namespace jinja2 wrapped in inline namespace v2 (scripts/abi_namespace.py) +ae4649c98baaf1999e99096f7d39ef7a4dca2b8d diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index c7f97435f..876aeb361 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -30,6 +30,11 @@ jobs: - name: Check formatting of src/ and include/ run: git ls-files 'src/**.h' 'src/**.cpp' 'include/**.h' | xargs clang-format-18 --dry-run --Werror + # Every namespace jinja2 in src/ and include/ opens inline namespace v2, the ABI tag + # (docs/tasks/0076); the nm test checks the symbols, this also covers header-only code + - name: Check the inline ABI namespace + run: python3 scripts/abi_namespace.py --check + # test/ still has hand-aligned tables, so only lines touched by the pull request # are checked there. src/ and include/ are left to the whole-file step above: given # a vendored file, git clang-format reports it as emptied (.clang-format-ignore). diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index e2cdfc8f0..504d73be8 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -46,6 +46,11 @@ jobs: # Two rows also install the library and build a consumer against the installed package # at another standard (`package`, docs/tasks/0068): a static and a shared install, each # read through find_package and pkg-config. It is a check after the tests, not an axis. + # One shared row per standard builds RelWithDebInfo (`abi`) so that abidiff can compare + # the library's exported ABI with the previous one (docs/tasks/0076): the headers differ + # by standard, so the ABI is checked once per standard. For the pairwise design those rows + # count as Release (optimised); the remaining Release rows still meet every standard, + # compiler, linkage, binding and noderef value on their own. linux-build: name: C++${{ matrix.std }} ${{ matrix.compiler }} ${{ matrix.config }} shared=${{ matrix.shared }} json=${{ matrix.bindings }}${{ matrix.noderef == 'OFF' && ' noderef=OFF' || '' }} runs-on: ubuntu-24.04 @@ -56,14 +61,14 @@ jobs: include: - { std: 17, compiler: gcc-12, config: Release, shared: 'OFF', bindings: boost, noderef: 'OFF' } - { std: 17, compiler: gcc-14, config: Debug, shared: 'OFF', bindings: rapid } - - { std: 17, compiler: clang-18, config: Release, shared: 'ON', bindings: nlohmann } + - { std: 17, compiler: clang-18, config: RelWithDebInfo, shared: 'ON', bindings: nlohmann, abi: true } - { std: 17, compiler: clang-20, config: Release, shared: 'OFF', bindings: boost, package: 23 } - - { std: 20, compiler: gcc-12, config: Release, shared: 'ON', bindings: rapid } + - { std: 20, compiler: gcc-12, config: RelWithDebInfo, shared: 'ON', bindings: rapid, abi: true } - { std: 20, compiler: gcc-14, config: Release, shared: 'ON', bindings: nlohmann } - { std: 20, compiler: clang-18, config: Debug, shared: 'OFF', bindings: boost } - { std: 20, compiler: clang-20, config: Debug, shared: 'OFF', bindings: nlohmann, noderef: 'OFF' } - { std: 23, compiler: gcc-12, config: Debug, shared: 'OFF', bindings: nlohmann } - - { std: 23, compiler: gcc-14, config: Release, shared: 'ON', bindings: boost, noderef: 'OFF', package: 17 } + - { std: 23, compiler: gcc-14, config: RelWithDebInfo, shared: 'ON', bindings: boost, noderef: 'OFF', package: 17, abi: true } - { std: 23, compiler: clang-18, config: Release, shared: 'ON', bindings: rapid, noderef: 'OFF' } - { std: 23, compiler: clang-20, config: Debug, shared: 'ON', bindings: rapid } - { std: 26, compiler: gcc-14, config: Release, shared: 'OFF', bindings: boost } @@ -99,6 +104,47 @@ jobs: - name: Test run: ctest --test-dir .build --output-on-failure + # abidiff against the ABI of the previous master build (report only) or, once the RC + # commits abi/libjinja2cpp.so.2.cxx.abi, against that baseline: an incompatible + # change then fails unless SOVERSION is bumped and the baseline regenerated (0076) + - name: ABI dump + if: matrix.abi + run: | + sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends abigail-tools + mkdir -p .abi + abidw --headers-dir include --drop-private-types --no-show-locs \ + --suppressions abi/suppressions.ini \ + --out-file .abi/libjinja2cpp.so.2.cxx${{ matrix.std }}.abi .build/libjinja2cpp.so + + - name: Previous ABI dump + if: matrix.abi && github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: .abi-base + key: abi-cxx${{ matrix.std }}-${{ github.event.pull_request.base.sha }} + restore-keys: abi-cxx${{ matrix.std }}- + + - name: ABI report + if: matrix.abi + run: scripts/abi_report.sh ${{ matrix.std }} + + - name: Save ABI dump + if: matrix.abi && github.event_name == 'push' + run: mkdir -p .abi-base && cp .abi/*.abi .abi-base/ + + - if: matrix.abi && github.event_name == 'push' + uses: actions/cache/save@v4 + with: + path: .abi-base + key: abi-cxx${{ matrix.std }}-${{ github.sha }} + + # also when the report fails: its message points at this dump + - if: ${{ !cancelled() && matrix.abi }} + uses: actions/upload-artifact@v4 + with: + name: abi-cxx${{ matrix.std }} + path: .abi/ + - name: Installed package if: matrix.package run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f41b47bc..41733a4fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,6 +173,7 @@ set(JINJA2CPP_SOURCES src/statements.cpp src/string_converter_filter.cpp src/template.cpp + src/template_dict.cpp src/template_env.cpp src/template_parser.cpp src/tester_factories.cpp @@ -456,6 +457,12 @@ if (JINJA2CPP_BUILD_TESTS) add_test(NAME jinja2cpp_tests COMMAND jinja2cpp_tests --gtest_filter=-Parity*) # Differential corpus against Python Jinja2, see test/parity/README.md add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*) + # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076) + if (CMAKE_NM AND NOT MSVC) + add_test(NAME jinja2cpp_abi_namespace + COMMAND ${CMAKE_COMMAND} -DNM=${CMAKE_NM} -DLIBRARY=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/test/abi_symbols.cmake) + endif () if ("${JINJA2CPP_WITH_JSON_BINDINGS}" STREQUAL "boost") set(_bindings_define "-DJINJA2CPP_WITH_JSON_BINDINGS_BOOST") diff --git a/README.md b/README.md index 3b63ff5c1..037989d0d 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ The install carries a CMake package and a pkg-config file. With CMake, point target: ```cmake -find_package(jinja2cpp 1 REQUIRED) +find_package(jinja2cpp 2 REQUIRED) target_link_libraries(my_app PRIVATE jinja2cpp::jinja2cpp) ``` @@ -201,6 +201,32 @@ it in the default internal dependencies mode). The un-namespaced `jinja2cpp` tar `lib/cmake/jinja2cpp` (was `lib/jinja2cpp`), the static library in `lib` (was `lib/static`) and `jinja2cpp.pc` in `lib/pkgconfig` (was `share/pkgconfig`). Without CMake, use `pkg-config --cflags --libs jinja2cpp` (`--static` for a static build). + +### Compatibility and versioning + +- **Names.** The 1.x names that 2.0 could keep still compile, marked `[[deprecated]]`; + they are removed in 3.0. +- **ABI across majors.** Everything is declared in `jinja2::v2`, an inline namespace: you + keep writing `jinja2::Value`, but the major version is part of every mangled name, so a + program compiled against 1.x headers fails to link with a 2.x library (static or shared) + instead of misbehaving at run time. Never spell `v2` yourself. To declare Jinja2C++ types + without their definitions, include ``: a hand-written + `namespace jinja2 { class Value; }` declares a different, undefined class and makes later + uses ambiguous. `` includes the whole API except the JSON bindings. + If your code says `using namespace jinja2;` and has its own namespace-scope `v2`, qualify + one of them. Specialise the extension points inside `namespace jinja2 { ... }` or with their + qualified name (`template<> struct jinja2::Reflector`). A 1.x-style + `detail::Reflector` specialisation must use the qualified form + (`template<> struct jinja2::detail::Reflector`): MSVC does not look through the + inline namespace when `namespace detail` is reopened inside `namespace jinja2`, and declares + a second `jinja2::detail` instead. +- **ABI within 2.x.** The shared library's SONAME carries the ABI version + (`libjinja2cpp.so.2`, `libjinja2cpp.2.dylib`). It is bumped by any release that changes the + layout of a type you allocate (a `Settings` field, a `Value` alternative) or the signature + of an exported function, so a shared consumer is never loaded against an incompatible + library. A static consumer has no such guard and rebuilds after a minor upgrade; patch + releases keep the ABI. + ### Usage with conan.io dependency manager Jinja2C++ can be used as conan.io package. In this case, you should do the following steps: diff --git a/abi/suppressions.ini b/abi/suppressions.ini new file mode 100644 index 000000000..27f5f5e65 --- /dev/null +++ b/abi/suppressions.ini @@ -0,0 +1,22 @@ +# libabigail suppressions for the abidiff job (.github/workflows/linux-build.yml, docs/tasks/0076). +# Only vendored internals that no public signature exposes are suppressed. jinja2::v2::detail +# is not: inline functions in the public headers call exported detail functions, so those +# are ABI too. + +# jbcoe/value_types, vendored as jinja2::v2::detail::xyz; header-only templates, their +# layout reaches users only through Value, which is reported on its own +[suppress_function] + name_regexp = ^jinja2::v2::detail::xyz:: + +[suppress_variable] + name_regexp = ^jinja2::v2::detail::xyz:: + +# robin_hood.h and lexertk.h are private to src/ and hidden by -fvisibility=hidden +[suppress_type] + name_regexp = ^(robin_hood|lexertk):: + +[suppress_function] + name_regexp = ^(robin_hood|lexertk):: + +[suppress_variable] + name_regexp = ^(robin_hood|lexertk):: diff --git a/docs/dependencies.md b/docs/dependencies.md index fc5f47a53..2252efab7 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -45,7 +45,7 @@ source and committed), `system` (found on the machine), `ci` (GitHub Actions). | Boost | 1.92.0 | BSL-1.0 | mandatory | link | fetched | the nine libraries below | Yes, piece by piece (docs/tasks/0230) | | robin-hood-hashing | 3.11.5, modified | MIT | mandatory | internal | vendored | flat hash map for values and scopes, faster than `std::unordered_map` here | Keep the idea; upstream is archived (docs/tasks/0231) | | lexertk | unrecorded, modified | MIT | mandatory | internal | vendored | expression tokenizer | Keep; effectively forked code | -| value_types | 57f2e7a, modified | MIT | mandatory | public-api | vendored | `polymorphic`/`indirect` in `jinja2::detail::xyz`, behind `jinja2::types` (Value, GenericList) | Keep; `std::` at C++26 | +| value_types | 57f2e7a, modified | MIT | mandatory | public-api | vendored | `polymorphic`/`indirect` in `jinja2::v2::detail::xyz`, behind `jinja2::types` (Value, GenericList) | Keep; `std::` at C++26 | | Unicode tables | UCD 15.0.0 via Python 3.12 | Unicode-3.0 | mandatory | internal | generated | Python-identical `isprintable`, case mapping, digits without ICU | Data only; acquisition decision open in 0187/0186 | | HTML entities | Python 3.12 `html.entities` | PSF-2.0 | mandatory | internal | generated | MarkupSafe-compatible unescape | Data only; keep | | Boost.JSON | 1.92.0 | BSL-1.0 | alternative (default) | public-optional | fetched | parses `{% meta %}` JSON for `GetMetadata` | Yes: the heaviest backend for a niche feature | diff --git a/docs/parity.md b/docs/parity.md index 0021db676..b8e7f8f73 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -4,7 +4,7 @@ How far Jinja2C++ is from Python [Jinja2](https://jinja.palletsprojects.com/) 3. area, and which task in `docs/tasks/` closes each gap. Every statement marked with a case id is backed by the differential corpus in -`test/parity/` (1939 templates rendered by both engines, see +`test/parity/` (1995 templates rendered by both engines, see [test/parity/README.md](../test/parity/README.md)); `ctest -R parity` re-checks all of them. Statements in the last section (API level) are read from the headers and are not corpus-checked yet. @@ -20,24 +20,24 @@ Snapshot of `python3 test/parity/generate.py --report` (Jinja2 3.1.6, Oct 2026): |---|---|---|---|---|---|---|---|---|---| | autoescape | 124 | 122 | 1 | 1 | 0 | 0 | 0 | 0 | 0210, 0216 | | custom | 28 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | | -| errors | 97 | 93 | 0 | 2 | 2 | 0 | 0 | 0 | 0036, 0213 | -| filters | 309 | 303 | 4 | 2 | 0 | 0 | 0 | 0 | 0101, 0178, 0203, 0215 | +| errors | 103 | 100 | 0 | 2 | 1 | 0 | 0 | 0 | 0213 | +| filters | 328 | 322 | 4 | 2 | 0 | 0 | 0 | 0 | 0101, 0178, 0203, 0215 | | fuzz | 32 | 32 | 0 | 0 | 0 | 0 | 0 | 0 | | -| globals | 74 | 73 | 1 | 0 | 0 | 0 | 0 | 0 | 0212 | +| globals | 78 | 77 | 1 | 0 | 0 | 0 | 0 | 0 | 0212 | | i18n | 54 | 54 | 0 | 0 | 0 | 0 | 0 | 0 | | -| literals | 68 | 65 | 2 | 1 | 0 | 0 | 0 | 0 | 0015, 0036, 0041 | +| literals | 75 | 73 | 1 | 1 | 0 | 0 | 0 | 0 | 0015, 0041 | | loader | 168 | 162 | 5 | 1 | 0 | 0 | 0 | 0 | 0165, 0211, 0274 | -| methods | 96 | 93 | 2 | 1 | 0 | 0 | 0 | 0 | 0049 | -| operators | 118 | 115 | 0 | 3 | 0 | 0 | 0 | 0 | 0015 | +| methods | 105 | 102 | 2 | 1 | 0 | 0 | 0 | 0 | 0049 | +| operators | 122 | 119 | 0 | 3 | 0 | 0 | 0 | 0 | 0015 | | options | 35 | 35 | 0 | 0 | 0 | 0 | 0 | 0 | | | output | 38 | 38 | 0 | 0 | 0 | 0 | 0 | 0 | | | sequences | 47 | 47 | 0 | 0 | 0 | 0 | 0 | 0 | | -| statements | 394 | 379 | 10 | 3 | 2 | 0 | 0 | 0 | 0156, 0165, 0209, 0214, 0219, 0220 | -| subscripts | 39 | 39 | 0 | 0 | 0 | 0 | 0 | 0 | | +| statements | 398 | 383 | 10 | 3 | 2 | 0 | 0 | 0 | 0156, 0165, 0209, 0214, 0219, 0220 | +| subscripts | 42 | 42 | 0 | 0 | 0 | 0 | 0 | 0 | | | tests | 58 | 57 | 0 | 1 | 0 | 0 | 0 | 0 | 0101 | | undefined | 106 | 106 | 0 | 0 | 0 | 0 | 0 | 0 | | | whitespace | 54 | 54 | 0 | 0 | 0 | 0 | 0 | 0 | | -| **total** | **1939** | **1895** | **25** | **15** | **4** | **0** | **0** | **0** | | +| **total** | **1995** | **1953** | **24** | **15** | **3** | **0** | **0** | **0** | | *output*: both render, text differs. *rejects*: C++ errors on a valid template. *accepts*: C++ renders a template Jinja2 rejects. *unsupported*: needs an Environment diff --git a/docs/tasks/0036-non-string-mapping-keys.md b/docs/tasks/0036-non-string-mapping-keys.md index 7bc9e2422..f234728aa 100644 --- a/docs/tasks/0036-non-string-mapping-keys.md +++ b/docs/tasks/0036-non-string-mapping-keys.md @@ -1,5 +1,5 @@ --- -status: open +status: done priority: low area: parity depends: [0013] @@ -26,3 +26,26 @@ tuple), which touches the public API and is worth doing only together with 0031. **Release decision (Ruslan 2026-10-18, D13 in docs/release-2.0.md).** In 2.0, template-internal only: the public `ValuesMap` key type stays `std::string` unless the architect plan shows a public need. + +**Shipped (D13 = b).** Dict literals, `dict()`, `namespace()` and the results of the dict +methods take any hashable key (`src/template_dict.*`): `1`, `1.0` and `True` are one key +(first spelling and position kept, last value wins), so are `(1, 2.0)` and `(1.0, 2)`; +`none`, floats, tuples and ranges are keys; a list or a dict raises `TypeError: unhashable +type`. Subscript, `in`, `get`, `setdefault`, `update`, `pop`, `popitem`, `copy`, iteration, +`items`, `dictsort`, `pprint`, `tojson`, `xmlattr`, `urlencode` and `==` follow Python. The +public `ValuesMap`/`GenericMap` keys stay `std::string`. + +*Conversion rule at the public boundary* (`CreateGenericMap` of a template dict, what a user +callable receives): a dict with a key that is not a string is handed over as a snapshot, not a +live view, keyed by `str(key)` (`{1: 'x', None: 'y'}` has the keys `"1"` and `"None"`). When two +keys spell alike (`{1: 'x', '1': 'y'}`) the later value wins at the position of the first, as +Python's `{str(k): v for k, v in d.items()}` has it. A dict of string keys stays a live view. +Changing this rule later is a break for callers who rely on it. + +*Not done:* object keys (`{namespace(): 1}`, host objects, callables, undefined) have no +identity token yet and all read as the empty string when stored, as before, while a lookup by +one finds nothing (`nothing in {'': 1}` is False); that is the first bullet of +0212 (`globals.object_dict_key` stays listed). All NaN keys are one key (Python compares NaN by +identity). +The render's cycle collector (0110) follows the kept keys too, so a cycle through a key +(`{% set d = {(ns, 1): 1} %}{% set ns.d = d %}`) is freed when the render ends. diff --git a/docs/tasks/0076-abi-namespace-and-headers.md b/docs/tasks/0076-abi-namespace-and-headers.md index cbd7b3bf9..6ad4b1fbf 100644 --- a/docs/tasks/0076-abi-namespace-and-headers.md +++ b/docs/tasks/0076-abi-namespace-and-headers.md @@ -1,9 +1,9 @@ --- -status: in-progress +status: done priority: high area: release depends: [0072, 0073, 0074, 0075, 0043] -touches: [include/, src/, CMakeLists.txt, cmake/public/, test/package/, test/fwd_test.cpp, test/value_ptr_test.cpp, README.md, sbom/, scripts/abi_namespace.py, .git-blame-ignore-revs, .github/workflows/format-check.yml] +touches: [include/, src/, CMakeLists.txt, cmake/public/, test/package/, test/fwd_test.cpp, test/value_ptr_test.cpp, test/abi_symbols.cmake, test/template_api_test.cpp, README.md, sbom/, abi/, scripts/abi_namespace.py, scripts/abi_report.sh, .git-blame-ignore-revs, .github/workflows/format-check.yml, .github/workflows/linux-build.yml] --- # 2.0: inline ABI namespace, header layout, version @@ -36,3 +36,19 @@ future 2.x build is binary-compatible. Settle the API-shaping decisions listed i script-generated `inline namespace v2` commit, the nm/link-probe/SONAME gates, the abidiff job and the README policy. Follow-ups: 0259 (robin_hood/lexertk namespaces), 0260 (``), 0261 (Windows package link check). + +**Done.** PR-1 #580. PR-2: `scripts/abi_namespace.py` wrote `inline namespace v2` into +every `namespace jinja2` of include/ and src/ (one commit, in .git-blame-ignore-revs) and +checks it in the format-check job; CTest `jinja2cpp_abi_namespace` (test/abi_symbols.cmake) +fails on a jinja2 symbol outside `v2` or a global `xyz`; test/package links +`abi_probe_v2` and expects `abi_probe_v1` (1.x mangling) not to link, in the static and +shared package rows; check_package.cmake checks the SONAME; the version asserts require +major 2; README "Compatibility and versioning". The abidiff job runs on one shared +RelWithDebInfo row per standard and reports against the last master build; committing the +baselines at the RC is 0285. Deliberate divergence from the plan: `jinja2::v2::detail` is +not suppressed in abidiff (exported detail functions are called from inline headers), and +the C++20 abi row is gcc-12, not gcc-14, so that the Release rows keep every pair; the C++17 +abi row stays clang-18 (the plan preferred GCC, but the only C++17 shared row is clang-18). +The gate counts any change left after `--no-added-syms` as a break, since abidiff reports +a changed layout (a new `Settings` field) with bit 4, not bit 8. The four Unicode/HTML +generators in scripts/ emit the `v2` form, so their `--check` stays green. diff --git a/docs/tasks/0285-abi-baselines-at-rc.md b/docs/tasks/0285-abi-baselines-at-rc.md new file mode 100644 index 000000000..b842a200d --- /dev/null +++ b/docs/tasks/0285-abi-baselines-at-rc.md @@ -0,0 +1,31 @@ +--- +status: open +priority: high +area: release +depends: [0076, 0221] +touches: [abi/, abi/suppressions.ini] +--- +# Commit the ABI baselines at the 2.0 RC and turn the abidiff gate on + +**Problem.** Since 0076 the three `abi` rows of linux-build.yml (one shared RelWithDebInfo +row per standard) dump the library's ABI with abidw and compare it with the last master +build. Until baselines are committed the job only reports: the public API is still +changing in the API wave, and failing on every change would block it. The SOVERSION +policy (README, "Compatibility and versioning") is enforced only once +`abi/libjinja2cpp.so.2.cxx{17,20,23}.abi` exist; then `scripts/abi_report.sh` fails an +incompatible change unless SOVERSION was bumped. + +**Proposal.** On the RC commit (0221 G1 freeze), download the `abi-cxx17`, `abi-cxx20` and +`abi-cxx23` artifacts of that commit's linux-build run and commit them as `abi/`. Before +that, read one report and review `abi/suppressions.ini`: it suppresses only the vendored +value_types and robin_hood/lexertk names; `jinja2::v2::detail` stays checked because +inline header code calls exported detail functions. Add suppressions only for noise that +is shown not to reach users. Two things to settle first: a RelWithDebInfo dump is about 31 MB +(1.2 MB gzipped), so commit the baselines compressed or trimmed and teach +`scripts/abi_report.sh` to read them; and the dump includes weak std and Boost symbols the +library exports (`boost::system` among them), so a Boost pin bump may read as a break +unless those are suppressed or hidden. `scripts/**` is in linux-build's `paths-ignore`, so +a PR that touches only `abi_report.sh` runs no abi row: check such a change by hand. + +**Done when.** The baselines are on master, a deliberate test change to an exported +layout makes the abi rows red, and docs/release-2.0.md lists the step in its RC procedure. diff --git a/docs/tasks/README.md b/docs/tasks/README.md index cf529ee40..08c571747 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -107,7 +107,7 @@ files go under `touches` (0012 and 0024 each rewrite about a hundred rows). | [0033](0033-wide-string-parity.md) | Run the corpus through the wide-string API | parity | low | done | | [0034](0034-none-versus-undefined.md) | Tell `None` apart from undefined | parity | high | done | | [0035](0035-locale-independent-string-conversion.md) | Convert between narrow and wide strings without the C locale | robustness | medium | done | -| [0036](0036-non-string-mapping-keys.md) | Mapping keys that are not strings | parity | low | open | +| [0036](0036-non-string-mapping-keys.md) | Mapping keys that are not strings | parity | low | done | | [0037](0037-sequence-protocol-follow-ups.md) | Sequence protocol follow-ups from 0016 | parity | medium | done | | [0038](0038-lexical-scoping-for-macros.md) | Lexical scoping for macros | parity | medium | done | | [0039](0039-float-values-print-widened.md) | C++ `float` values print with their widened digits | parity | low | open | @@ -145,7 +145,7 @@ files go under `touches` (0012 and 0024 each rewrite about a hundred rows). | [0073](0073-template-api-2-0.md) | 2.0 API: `BasicTemplate` | release | medium | done | | [0074](0074-template-env-api-2-0.md) | 2.0 API: `TemplateEnv` pimpl and `Settings` | release | medium | done | | [0075](0075-containers-reflection-api-2-0.md) | 2.0 API: containers, reflection, errors | release | medium | done | -| [0076](0076-abi-namespace-and-headers.md) | 2.0: inline ABI namespace, header layout, version | release | high | in-progress | +| [0076](0076-abi-namespace-and-headers.md) | 2.0: inline ABI namespace, header layout, version | release | high | done | | [0077](0077-migration-script.md) | 2.0 migration script and notes | release | high | open | | [0078](0078-nix-toolchains.md) | Pinned toolchains from Nix for the tool and bleeding-edge rows | ci | low | open | | [0079](0079-msvc-runtime-shared-override.md) | Warn when `JINJA2CPP_MSVC_RUNTIME_TYPE` is overridden by a shared build | build | low | open | @@ -345,3 +345,4 @@ files go under `touches` (0012 and 0024 each rewrite about a hundred rows). | [0280](0280-urlize-markup-input.md) | urlize escapes Markup input | filters | low | open | | [0281](0281-attr-bound-method.md) | attr() of a method renders empty | filters | low | open | | [0284](0284-template-module.md) | `Template.module` / `make_module`: a template's exports as a map | api | low | open | +| [0285](0285-abi-baselines-at-rc.md) | Commit the ABI baselines at the 2.0 RC and turn the abidiff gate on | release | high | open | diff --git a/include/jinja2cpp/big_int.h b/include/jinja2cpp/big_int.h index 8b1a42f8a..b29b14f84 100644 --- a/include/jinja2cpp/big_int.h +++ b/include/jinja2cpp/big_int.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { //! Integral types a \ref BigInt is made from: all except `bool` and the character types, as for \ref Value @@ -204,6 +206,7 @@ class BigInt std::string m_decimal = "0"; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_BIG_INT_H diff --git a/include/jinja2cpp/binding/boost_json.h b/include/jinja2cpp/binding/boost_json.h index 61c2717b7..3733d329b 100644 --- a/include/jinja2cpp/binding/boost_json.h +++ b/include/jinja2cpp/binding/boost_json.h @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -247,6 +249,7 @@ struct Reflector } }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_BINDING_BOOST_JSON_H diff --git a/include/jinja2cpp/binding/nlohmann_json.h b/include/jinja2cpp/binding/nlohmann_json.h index 6a8e7b786..8abeb8ae8 100644 --- a/include/jinja2cpp/binding/nlohmann_json.h +++ b/include/jinja2cpp/binding/nlohmann_json.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -237,6 +239,7 @@ struct Reflector : detail::NLohmannJsonReflector> return GenericMap([accessor = detail::RapidJsonObjectAccessor>(val)]() { return &accessor; }); } }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_BINDING_RAPID_JSON_H diff --git a/include/jinja2cpp/error_info.h b/include/jinja2cpp/error_info.h index faa989d33..af5f4dc2e 100644 --- a/include/jinja2cpp/error_info.h +++ b/include/jinja2cpp/error_info.h @@ -14,6 +14,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ /*! * \brief Type of the error * @@ -197,6 +199,7 @@ using ErrorInfoTpl JINJA2CPP_DEPRECATED("jinja2cpp-2: BasicErrorInfo") = BasicEr JINJA2CPP_EXPORT std::ostream& operator<<(std::ostream& os, const ErrorInfo& res); JINJA2CPP_EXPORT std::wostream& operator<<(std::wostream& os, const ErrorInfoW& res); +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/filesystem_handler.h b/include/jinja2cpp/filesystem_handler.h index f5b0556c5..f89ffc02b 100644 --- a/include/jinja2cpp/filesystem_handler.h +++ b/include/jinja2cpp/filesystem_handler.h @@ -19,6 +19,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ template using FileStreamPtr = std::unique_ptr, void (*)(std::basic_istream*)>; @@ -207,6 +209,7 @@ class JINJA2CPP_EXPORT RealFileSystem : public IFilesystemHandler private: std::string m_rootFolder; }; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/fwd.h b/include/jinja2cpp/fwd.h index ae7a3cd75..46844153c 100644 --- a/include/jinja2cpp/fwd.h +++ b/include/jinja2cpp/fwd.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ class Value; class GenericList; class GenericMap; @@ -59,6 +61,7 @@ using ErrorInfo = BasicErrorInfo; using ErrorInfoW = BasicErrorInfo; using ValuesList = std::vector; using FilesystemHandlerPtr = std::shared_ptr; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_FWD_H diff --git a/include/jinja2cpp/generic_list.h b/include/jinja2cpp/generic_list.h index 86622e1ee..012624cbc 100644 --- a/include/jinja2cpp/generic_list.h +++ b/include/jinja2cpp/generic_list.h @@ -18,6 +18,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ /*! * \brief Interface for accessing items in list by the indexes * @@ -287,6 +289,7 @@ inline ListEnumeratorPtr IListItemAccessor::MakeEnumerator(Args&&... args) { return ListEnumeratorPtr{ types::in_place_type_t{}, std::forward(args)... }; } +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/generic_list_iterator.h b/include/jinja2cpp/generic_list_iterator.h index 45eab1f69..d5b017568 100644 --- a/include/jinja2cpp/generic_list_iterator.h +++ b/include/jinja2cpp/generic_list_iterator.h @@ -15,7 +15,11 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { class JINJA2CPP_EXPORT GenericListIterator { @@ -124,7 +128,9 @@ class JINJA2CPP_EXPORT GenericListIterator Value m_current; }; -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/make_generic_list.h b/include/jinja2cpp/make_generic_list.h index f1f50dc2e..17e45a473 100644 --- a/include/jinja2cpp/make_generic_list.h +++ b/include/jinja2cpp/make_generic_list.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ /*! * \brief Generator of a generated list, see \ref MakeGenericList(ListGenerator) * @@ -513,6 +515,7 @@ inline GenericList MakeGenericList(ListGenerator fn) return detail::MakeGeneratedList(std::move(fn)); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_MAKE_GENERIC_LIST_H diff --git a/include/jinja2cpp/markup.h b/include/jinja2cpp/markup.h index 7e956ad1f..19dae86db 100644 --- a/include/jinja2cpp/markup.h +++ b/include/jinja2cpp/markup.h @@ -8,6 +8,8 @@ namespace jinja2 { +inline namespace v2 +{ /*! * \brief Text that is already safe HTML: Python's `markupsafe.Markup` * @@ -115,6 +117,7 @@ class BasicMarkup using Markup = BasicMarkup; //! Safe wide text, see \ref BasicMarkup using WMarkup = BasicMarkup; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_MARKUP_H diff --git a/include/jinja2cpp/polymorphic_value/indirect_cxx14.h b/include/jinja2cpp/polymorphic_value/indirect_cxx14.h index 96efe3106..89b27cdf4 100644 --- a/include/jinja2cpp/polymorphic_value/indirect_cxx14.h +++ b/include/jinja2cpp/polymorphic_value/indirect_cxx14.h @@ -30,12 +30,13 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // Local change: upstream includes "feature_check.h" here for // XYZ_HAS_TEMPLATE_ARGUMENT_DEDUCTION; the C++17 floor always has it, and this // copy stays a single header. Vendored from jbcoe/value_types 57f2e7a (task 0072). -// Local change (task 0076): namespace xyz moved into jinja2::detail and the +// Local change (task 0076): namespace xyz moved into jinja2::v2::detail and the // XYZ_ macros renamed JINJA2CPP_XYZ_, so a user's own copy of value_types // neither shadows this one through the shared include guards nor collides // with it at link time. namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { #ifndef JINJA2CPP_XYZ_IN_PLACE_DEFINED @@ -616,6 +617,7 @@ indirect(std::allocator_arg_t, Alloc, Value) -> indirect< } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 template diff --git a/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h b/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h index b41d702a0..489ae76f9 100644 --- a/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h +++ b/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h @@ -29,7 +29,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // Vendored from jbcoe/value_types 57f2e7a (compatibility/polymorphic_cxx14.h) // with these local changes: -// - namespace xyz moved into jinja2::detail and the XYZ_ macros renamed +// - namespace xyz moved into jinja2::v2::detail and the XYZ_ macros renamed // JINJA2CPP_XYZ_ (task 0076), so a user's own copy of value_types neither // shadows this one through the shared include guards nor collides with it at // link time; @@ -47,12 +47,14 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define JINJA2CPP_XYZ_IN_PLACE_TYPE_DEFINED namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { template struct in_place_type_t {}; } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_IN_PLACE_TYPE_DEFINED @@ -60,6 +62,7 @@ struct in_place_type_t {}; #define JINJA2CPP_XYZ_UNREACHABLE_DEFINED namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { [[noreturn]] inline void unreachable() { // LCOV_EXCL_LINE @@ -73,6 +76,7 @@ namespace xyz { } } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_UNREACHABLE_DEFINED @@ -85,6 +89,7 @@ namespace xyz { // We duplicate implementations to allow this header to work as a single // include. https://godbolt.org needs single-file includes. namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { namespace detail { @@ -121,10 +126,12 @@ class empty_base_optimization : private T { } // namespace detail } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_EMPTY_BASE_DEFINED namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { @@ -505,6 +512,7 @@ class polymorphic : private detail::empty_base_optimization { } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_POLYMORPHIC_H_ diff --git a/include/jinja2cpp/reflected_value.h b/include/jinja2cpp/reflected_value.h index bc6dd30e4..cbb6ab951 100644 --- a/include/jinja2cpp/reflected_value.h +++ b/include/jinja2cpp/reflected_value.h @@ -23,6 +23,8 @@ namespace jinja2 { +inline namespace v2 +{ /*! * \brief Reflect the arbitrary C++ type to the Jinja2C++ engine @@ -687,6 +689,7 @@ Value Reflect(T&& val) return Reflector::Create(std::forward(val)); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_REFLECTED_VALUE_H diff --git a/include/jinja2cpp/string_helpers.h b/include/jinja2cpp/string_helpers.h index b8d27868c..f626f830d 100644 --- a/include/jinja2cpp/string_helpers.h +++ b/include/jinja2cpp/string_helpers.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { template @@ -374,6 +376,7 @@ inline std::string AsString(const Value& val) { return std::visit(detail::String //! The stored string, converted if needed, or an empty string for any other value, as in 1.x; \ref ToWString prints every value JINJA2CPP_DEPRECATED("jinja2cpp-2: ToWString") inline std::wstring AsWString(const Value& val) { return std::visit(detail::WStringGetter(), val.GetData()); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_STRING_HELPERS_H diff --git a/include/jinja2cpp/template.h b/include/jinja2cpp/template.h index fec641b58..fd88747ae 100644 --- a/include/jinja2cpp/template.h +++ b/include/jinja2cpp/template.h @@ -28,6 +28,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ class ITemplateImpl; template class TemplateImpl; @@ -247,6 +249,7 @@ using Template = BasicTemplate; //! Template with wide (`wchar_t`) source and output using TemplateW = BasicTemplate; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/template_env.h b/include/jinja2cpp/template_env.h index 918923270..a787cbb56 100644 --- a/include/jinja2cpp/template_env.h +++ b/include/jinja2cpp/template_env.h @@ -22,6 +22,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -379,6 +381,7 @@ class JINJA2CPP_EXPORT TemplateEnv std::shared_ptr m_impl; }; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/user_callable.h b/include/jinja2cpp/user_callable.h index 78b152e4b..b98d87a38 100644 --- a/include/jinja2cpp/user_callable.h +++ b/include/jinja2cpp/user_callable.h @@ -17,6 +17,8 @@ namespace jinja2 { +inline namespace v2 +{ #ifndef JINJA2CPP_NO_DOXYGEN namespace detail { @@ -455,6 +457,7 @@ auto MakeCallable(Fn&& f) { return UserCallable{ [=, fn = std::forward(f)](const UserCallableParams&) { return fn(); }, {} }; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_USER_CALLABLE_H diff --git a/include/jinja2cpp/utils/i_comparable.h b/include/jinja2cpp/utils/i_comparable.h index 2f1cbc6de..0860ca1a2 100644 --- a/include/jinja2cpp/utils/i_comparable.h +++ b/include/jinja2cpp/utils/i_comparable.h @@ -8,6 +8,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ /*! * \brief Equality comparison for the extension interfaces (list and map accessors, enumerators, filesystem handlers) @@ -28,6 +30,7 @@ struct JINJA2CPP_EXPORT IComparable [[nodiscard]] virtual bool IsEqual(const IComparable& other) const { return this == &other; } }; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/utils/ordered_map.h b/include/jinja2cpp/utils/ordered_map.h index 6e53b9418..3d120c447 100644 --- a/include/jinja2cpp/utils/ordered_map.h +++ b/include/jinja2cpp/utils/ordered_map.h @@ -17,6 +17,8 @@ namespace jinja2 { +inline namespace v2 +{ // A hash map that iterates in insertion order, like a Python dict (docs/tasks/0031, 0043). It is the base of // the public ValuesMap and of the library's own dicts. @@ -408,6 +410,7 @@ class OrderedMap std::unordered_map m_index; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_UTILS_ORDERED_MAP_H diff --git a/include/jinja2cpp/value.h b/include/jinja2cpp/value.h index 2ace125f1..30e48c0ac 100644 --- a/include/jinja2cpp/value.h +++ b/include/jinja2cpp/value.h @@ -31,6 +31,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ //! Python `None`: what a default-constructed \ref Value holds struct NoneValue { @@ -1368,6 +1370,7 @@ decltype(auto) Visit(Fn&& fn, Value& val) return detail::VisitUnwrapped(std::forward(fn), val.GetData()); } +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/value_ptr.h b/include/jinja2cpp/value_ptr.h index debdb2d43..6638a919b 100644 --- a/include/jinja2cpp/value_ptr.h +++ b/include/jinja2cpp/value_ptr.h @@ -4,7 +4,11 @@ #include "jinja2cpp/polymorphic_value/indirect_cxx14.h" #include "jinja2cpp/polymorphic_value/polymorphic_cxx14.h" -namespace jinja2::types +namespace jinja2 +{ +inline namespace v2 +{ +namespace types { // Only the names the library's headers use, not the whole vendored namespace @@ -20,9 +24,15 @@ auto MakeValuePtr(Ts&&... ts) return polymorphic(in_place_type_t{}, std::forward(ts)...); } -} // namespace jinja2::types +} // namespace types +} // namespace v2 +} // namespace jinja2 -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // Owns one heap-allocated T and copies it deeply, as C++26 std::indirect does; T may be // incomplete where the wrapper is declared. Holds the recursive alternatives of Value. @@ -30,6 +40,8 @@ template using indirect = xyz::indirect; // The tag that selects indirect's in-place constructor using in_place_t = xyz::in_place_t; -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_VALUE_PTR_H diff --git a/sbom/components.json b/sbom/components.json index 1a33e1033..1d94569ac 100644 --- a/sbom/components.json +++ b/sbom/components.json @@ -278,12 +278,12 @@ "purl": "pkg:github/jbcoe/value_types@57f2e7a", "delivery": "vendored", "files": ["include/jinja2cpp/polymorphic_value/indirect_cxx14.h", "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h"], - "modified": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T", + "modified": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T", "role": "mandatory", "phases": ["build", "consumer-build"], "exposure": "public-api", "selected_by": "always", - "used_for": "polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap", + "used_for": "polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap", "why": "Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26", "lighten": "Keep (header-only, installed). Switch to std:: at a C++26 floor" }, @@ -571,6 +571,24 @@ "lighten": "Not part of any build", "evidence": [{"file": ".github/workflows/format-check.yml", "contains": ["clang-format-18"]}] }, + { + "id": "libabigail", + "name": "libabigail (abigail-tools)", + "version": "2.4", + "license": "Apache-2.0 WITH LLVM-exception", + "supplier": "Red Hat / sourceware.org", + "url": "https://sourceware.org/libabigail/", + "purl": "pkg:generic/libabigail@2.4", + "delivery": "system", + "role": "tooling", + "phases": ["ci"], + "exposure": "none", + "selected_by": "linux-build CI abi rows (apt abigail-tools, Ubuntu 24.04)", + "used_for": "ABI dump and comparison of the shared library (abidw, abidiff)", + "why": "SOVERSION policy gate (docs/tasks/0076)", + "lighten": "Not part of any build", + "evidence": [{"file": ".github/workflows/linux-build.yml", "contains": ["abigail-tools"]}] + }, { "id": "cmake", "name": "CMake", diff --git a/sbom/jinja2cpp-full.cdx.json b/sbom/jinja2cpp-full.cdx.json index f241ed167..5f9660898 100644 --- a/sbom/jinja2cpp-full.cdx.json +++ b/sbom/jinja2cpp-full.cdx.json @@ -1,7 +1,7 @@ { "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:3496cdbc-cf00-54e8-9a51-38efe8127a5d", + "serialNumber": "urn:uuid:a7ddc095-8bb8-5764-b9ea-9b60eb28ef13", "version": 1, "metadata": { "timestamp": "2026-10-18T00:00:00Z", @@ -1122,7 +1122,7 @@ } ], "pedigree": { - "notes": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "notes": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, "properties": [ { @@ -1147,7 +1147,7 @@ }, { "name": "jinja2cpp:used-for", - "value": "polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" + "value": "polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" }, { "name": "jinja2cpp:why", @@ -1159,15 +1159,15 @@ }, { "name": "jinja2cpp:modified", - "value": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "value": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3" + "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a" }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" } ] }, @@ -1229,15 +1229,15 @@ }, { "name": "jinja2cpp:file", - "value": "src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc" + "value": "src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d" }, { "name": "jinja2cpp:file", - "value": "src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b" + "value": "src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0" }, { "name": "jinja2cpp:file", - "value": "src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3" + "value": "src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243" }, { "name": "jinja2cpp:generator", @@ -1265,7 +1265,7 @@ "hashes": [ { "alg": "SHA-256", - "content": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "content": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "licenses": [ @@ -1317,7 +1317,7 @@ }, { "name": "jinja2cpp:file", - "value": "src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "value": "src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" }, { "name": "jinja2cpp:generator", @@ -2093,6 +2093,62 @@ } ] }, + { + "type": "application", + "bom-ref": "libabigail", + "name": "libabigail (abigail-tools)", + "version": "2.4", + "supplier": { + "name": "Red Hat / sourceware.org" + }, + "scope": "excluded", + "licenses": [ + { + "expression": "Apache-2.0 WITH LLVM-exception" + } + ], + "purl": "pkg:generic/libabigail@2.4", + "externalReferences": [ + { + "type": "website", + "url": "https://sourceware.org/libabigail/" + } + ], + "properties": [ + { + "name": "jinja2cpp:role", + "value": "tooling" + }, + { + "name": "jinja2cpp:phases", + "value": "ci" + }, + { + "name": "jinja2cpp:exposure", + "value": "none" + }, + { + "name": "jinja2cpp:delivery", + "value": "system" + }, + { + "name": "jinja2cpp:selected-by", + "value": "linux-build CI abi rows (apt abigail-tools, Ubuntu 24.04)" + }, + { + "name": "jinja2cpp:used-for", + "value": "ABI dump and comparison of the shared library (abidw, abidiff)" + }, + { + "name": "jinja2cpp:why", + "value": "SOVERSION policy gate (docs/tasks/0076)" + }, + { + "name": "jinja2cpp:lighten", + "value": "Not part of any build" + } + ] + }, { "type": "application", "bom-ref": "cmake", @@ -2710,6 +2766,7 @@ "clang-tidy", "pyyaml", "clang-format", + "libabigail", "cmake", "conan", "lars-bilke-codecoverage", diff --git a/sbom/jinja2cpp-full.spdx.json b/sbom/jinja2cpp-full.spdx.json index 1af5aa90a..9bf8e4c0c 100644 --- a/sbom/jinja2cpp-full.spdx.json +++ b/sbom/jinja2cpp-full.spdx.json @@ -427,7 +427,7 @@ } ], "primaryPackagePurpose": "SOURCE", - "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" }, { "SPDXID": "SPDXRef-Package-unicode-data", @@ -448,7 +448,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc; file: src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b; file: src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d; file: src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0; file: src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" }, { "SPDXID": "SPDXRef-Package-html-entities", @@ -460,7 +460,7 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "checksumValue": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "homepage": "https://docs.python.org/3.12/library/html.entities.html", @@ -475,7 +475,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e; generator: scripts/gen_html_entities.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6; generator: scripts/gen_html_entities.py" }, { "SPDXID": "SPDXRef-Package-googletest", @@ -748,6 +748,27 @@ "primaryPackagePurpose": "APPLICATION", "comment": "role: tooling; phases: dev; exposure: none; delivery: system; selected-by: format-check CI job (apt clang-format-18); used-for: Formatting gate; why: Code style; lighten: Not part of any build" }, + { + "SPDXID": "SPDXRef-Package-libabigail", + "name": "libabigail (abigail-tools)", + "versionInfo": "2.4", + "supplier": "Organization: Red Hat / sourceware.org", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "homepage": "https://sourceware.org/libabigail/", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Apache-2.0 WITH LLVM-exception", + "copyrightText": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libabigail@2.4" + } + ], + "primaryPackagePurpose": "APPLICATION", + "comment": "role: tooling; phases: ci; exposure: none; delivery: system; selected-by: linux-build CI abi rows (apt abigail-tools, Ubuntu 24.04); used-for: ABI dump and comparison of the shared library (abidw, abidiff); why: SOVERSION policy gate (docs/tasks/0076); lighten: Not part of any build" + }, { "SPDXID": "SPDXRef-Package-cmake", "name": "CMake", @@ -1130,6 +1151,11 @@ "relationshipType": "DEV_TOOL_OF", "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" }, + { + "spdxElementId": "SPDXRef-Package-libabigail", + "relationshipType": "DEV_TOOL_OF", + "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" + }, { "spdxElementId": "SPDXRef-Package-cmake", "relationshipType": "BUILD_TOOL_OF", @@ -1181,5 +1207,5 @@ "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" } ], - "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-full-6aee0b571f4d138f" + "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-full-45343781f625a58f" } diff --git a/sbom/jinja2cpp.cdx.json b/sbom/jinja2cpp.cdx.json index 258ac8517..e35ee8cd0 100644 --- a/sbom/jinja2cpp.cdx.json +++ b/sbom/jinja2cpp.cdx.json @@ -1,7 +1,7 @@ { "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:11349e2e-8f15-5505-bdfe-c51fdcc762fb", + "serialNumber": "urn:uuid:6517a686-4ed3-5d9a-9ee7-94f64aa14c22", "version": 1, "metadata": { "timestamp": "2026-10-18T00:00:00Z", @@ -1122,7 +1122,7 @@ } ], "pedigree": { - "notes": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "notes": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, "properties": [ { @@ -1147,7 +1147,7 @@ }, { "name": "jinja2cpp:used-for", - "value": "polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" + "value": "polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" }, { "name": "jinja2cpp:why", @@ -1159,15 +1159,15 @@ }, { "name": "jinja2cpp:modified", - "value": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "value": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3" + "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a" }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" } ] }, @@ -1229,15 +1229,15 @@ }, { "name": "jinja2cpp:file", - "value": "src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc" + "value": "src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d" }, { "name": "jinja2cpp:file", - "value": "src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b" + "value": "src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0" }, { "name": "jinja2cpp:file", - "value": "src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3" + "value": "src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243" }, { "name": "jinja2cpp:generator", @@ -1265,7 +1265,7 @@ "hashes": [ { "alg": "SHA-256", - "content": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "content": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "licenses": [ @@ -1317,7 +1317,7 @@ }, { "name": "jinja2cpp:file", - "value": "src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "value": "src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" }, { "name": "jinja2cpp:generator", diff --git a/sbom/jinja2cpp.spdx.json b/sbom/jinja2cpp.spdx.json index 25e53833f..fe2683991 100644 --- a/sbom/jinja2cpp.spdx.json +++ b/sbom/jinja2cpp.spdx.json @@ -427,7 +427,7 @@ } ], "primaryPackagePurpose": "SOURCE", - "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" }, { "SPDXID": "SPDXRef-Package-unicode-data", @@ -448,7 +448,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc; file: src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b; file: src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d; file: src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0; file: src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" }, { "SPDXID": "SPDXRef-Package-html-entities", @@ -460,7 +460,7 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "checksumValue": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "homepage": "https://docs.python.org/3.12/library/html.entities.html", @@ -475,7 +475,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e; generator: scripts/gen_html_entities.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6; generator: scripts/gen_html_entities.py" } ], "relationships": [ @@ -580,5 +580,5 @@ "relatedSpdxElement": "SPDXRef-Package-html-entities" } ], - "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-library-5b4dd302002fa2d1" + "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-library-751d894c115a1ae6" } diff --git a/scripts/abi_namespace.py b/scripts/abi_namespace.py new file mode 100755 index 000000000..b400f1c30 --- /dev/null +++ b/scripts/abi_namespace.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Keep every `namespace jinja2` in include/ and src/ inside the inline ABI namespace (docs/tasks/0076). + + scripts/abi_namespace.py # rewrite the files in place + scripts/abi_namespace.py --check # list what would change, exit 1 if anything would + +Public and internal code live in `jinja2::v2`, an inline namespace: users still write +`jinja2::Value`, but the mangled names carry the major version, so a program built against +1.x headers fails to link with a 2.x library instead of misbehaving. The tag is spelled out +in every file rather than hidden behind a macro (house clang-format and clang-tidy handle the +literal form); this script writes it so that nobody has to: + + namespace jinja2 namespace jinja2 + { { + ... -> inline namespace v2 + } { + ... + } // namespace v2 + } + +A nested `namespace jinja2::detail` becomes `namespace jinja2 { inline namespace v2 { +namespace detail` (C++17 cannot nest an inline namespace in that form). An opening already +followed by `inline namespace vN` with another N gets the new tag, so the 3.0 retag is one +run with TAG changed. The vendored polymorphic_value headers are rewritten in their own +upstream brace style; robin_hood.h and lexertk.h do not use namespace jinja2. +""" + +import argparse +import os +import re +import subprocess +import sys + +TAG = "v2" + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROOTS = ("include", "src") +OPENING = re.compile(r"^namespace jinja2(?P(?:::\w+)*)(?P\s*\{)?\s*$") +ANY_OPENING = re.compile(r"(?v\d+)(?P\s*\{)?\s*$") +CLOSE_COMMENT = re.compile(r"^\}(?P\s*)//\s*namespace\s+(?P[\w:]+)\s*$") + + +def code_mask(text): + """Copy of text with comments, string and character literals blanked, newlines kept.""" + out = list(text) + i, n = 0, len(text) + + def blank(a, b): + for k in range(a, b): + if out[k] != "\n": + out[k] = " " + + while i < n: + c = text[i] + if text.startswith("//", i): + j = text.find("\n", i) + j = n if j < 0 else j + blank(i, j) + i = j + elif text.startswith("/*", i): + j = text.find("*/", i + 2) + j = n if j < 0 else j + 2 + blank(i, j) + i = j + elif c == '"' or c == "'": + # R"delim( ... )delim", with any encoding prefix + prefix = re.search(r"(?:u8|[uUL])?R$", text[max(0, i - 3) : i]) + before = i - len(prefix.group(0)) - 1 if prefix else -1 + if c == '"' and prefix and (before < 0 or not (text[before].isalnum() or text[before] == "_")): + m = re.match(r'"([^(\s]*)\(', text[i:]) + if m: + end = text.find(")" + m.group(1) + '"', i + len(m.group(0))) + end = n if end < 0 else end + len(m.group(1)) + 2 + blank(i + 1, end - 1) + i = end + continue + if c == "'" and i > 0 and text[i - 1].isalnum() and i + 1 < n and text[i + 1].isalnum(): + i += 1 # digit separator: 1'000 + continue + j = i + 1 + while j < n and text[j] != c and text[j] != "\n": + j += 2 if text[j] == "\\" else 1 + blank(i + 1, min(j, n)) + i = j + 1 + else: + i += 1 + return "".join(out) + + +def matching_close(mask_lines, line, col): + """Line index of the `}` closing the `{` at (line, col) of the masked lines.""" + depth = 0 + for li in range(line, len(mask_lines)): + start = col if li == line else 0 + for ch in mask_lines[li][start:]: + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return li + raise ValueError("unbalanced braces") + + +def brace_after(mask_lines, line): + """(line, col) of the first `{` at or after the end of `line` (the opening's brace).""" + for li in range(line, len(mask_lines)): + col = mask_lines[li].find("{") + if col >= 0: + return li, col + if li > line and mask_lines[li].strip(): + break + raise ValueError("namespace opening without a brace") + + +def next_code_line(mask_lines, line): + for li in range(line + 1, len(mask_lines)): + if mask_lines[li].strip(): + return li + return None + + +def rewrite(text): + """Return text with every namespace jinja2 opening wrapped in `inline namespace TAG`.""" + lines = text.split("\n") + # Each pass fixes the first opening that needs it; the next pass re-reads the result. + for _ in range(10000): + mask_lines = code_mask("\n".join(lines)).split("\n") + edit = None + for li, raw in enumerate(lines): + m = OPENING.match(mask_lines[li].rstrip()) + if not m: + continue + same_line = bool(m.group("brace")) + open_line, open_col = brace_after(mask_lines, li) + close_line = matching_close(mask_lines, open_line, open_col) + close_text = lines[close_line] + cm = CLOSE_COMMENT.match(close_text.strip()) + space = cm.group("space") if cm else " " + if m.group("nested"): + inner = m.group("nested")[2:] + head = ["namespace jinja2", "{", f"inline namespace {TAG}", "{", f"namespace {inner}"] + if same_line: + head = ["namespace jinja2 {", f"inline namespace {TAG} {{", f"namespace {inner} {{"] + closing = "}" + space + f"// namespace {inner}" if cm else close_text + edit = (li, li + 1, head, close_line, [closing, "}" + space + f"// namespace {TAG}", "}" + space + "// namespace jinja2"]) + break + nxt = next_code_line(mask_lines, open_line if not same_line else li) + im = INLINE.match(mask_lines[nxt].strip()) if nxt is not None else None + if im: + if im.group("tag") == TAG: + continue + inline_brace_line, inline_col = brace_after(mask_lines, nxt) + inline_close = matching_close(mask_lines, inline_brace_line, inline_col) + lines[nxt] = lines[nxt].replace(im.group("tag"), TAG, 1) + icm = CLOSE_COMMENT.match(lines[inline_close].strip()) + if icm: + lines[inline_close] = "}" + icm.group("space") + f"// namespace {TAG}" + edit = () + break + insert = [f"inline namespace {TAG} {{"] if same_line else [f"inline namespace {TAG}", "{"] + edit = (open_line + 1, open_line + 1, insert, close_line, ["}" + space + f"// namespace {TAG}", close_text]) + break + if edit is None: + # Any other spelling (indented, one-line `namespace jinja2 { ... }`) would stay + # outside v2 unnoticed: a free function in it becomes a second overload + for li, code in enumerate(mask_lines): + if ANY_OPENING.search(code) and not OPENING.match(code.rstrip()): + raise ValueError(f"line {li + 1}: write `namespace jinja2` alone at the start of a line") + return "\n".join(lines) + if edit == (): + continue + a, b, new_open, close_line, new_close = edit + lines[close_line:close_line + 1] = new_close + lines[a:b] = new_open + raise RuntimeError("too many namespace openings") + + +def files(): + out = subprocess.run(["git", "ls-files", "--", *ROOTS], cwd=REPO, capture_output=True, text=True, check=True).stdout + for path in out.split(): + if path.endswith((".h", ".cpp")) and not path.endswith(("robin_hood.h", "lexertk.h")): + yield path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--check", action="store_true", help="change nothing; exit 1 if a file would change") + args = parser.parse_args() + + changed = [] + for path in files(): + full = os.path.join(REPO, path) + with open(full, encoding="utf-8", newline="") as f: + text = f.read() + try: + new = rewrite(text) + except ValueError as e: + print(f"{path}: {e}", file=sys.stderr) + return 2 + if new != text: + changed.append(path) + if not args.check: + with open(full, "w", encoding="utf-8", newline="") as f: + f.write(new) + + if args.check: + for path in changed: + print(f"{path}: namespace jinja2 not wrapped in inline namespace {TAG}") + if changed: + print("Run scripts/abi_namespace.py and commit the result (docs/tasks/0076).") + return 1 + return 0 + print(f"{len(changed)} files rewritten") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/abi_report.sh b/scripts/abi_report.sh new file mode 100755 index 000000000..1ea53fa57 --- /dev/null +++ b/scripts/abi_report.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Compares the library's ABI with the previous one for one C++ standard (docs/tasks/0076). +# scripts/abi_report.sh # after abidw wrote .abi/libjinja2cpp.so.2.cxx.abi +# Run by the abidiff steps of .github/workflows/linux-build.yml. +# +# Until the 2.0 RC no baseline is committed and the dump of the last master build +# (.abi-base/, restored from the Actions cache) is the reference: the report shows what a +# pull request changes and never fails. From the RC on, abi/libjinja2cpp.so..cxx.abi +# is the reference and the gate fails +# - on an incompatible change: any change left once added symbols are ignored (removed or +# changed functions and variables, changed layouts of the types they use). The way out is +# to bump SOVERSION and replace the baseline with the new dump in the same pull request; +# - when the baseline's SONAME differs from the library's, so that a SOVERSION bump cannot +# land without its baseline. +set -uo pipefail + +std=${1:?usage: abi_report.sh } +current=$(ls .abi/libjinja2cpp.so.*.cxx"$std".abi 2>/dev/null | head -n 1) +baseline=$(ls abi/libjinja2cpp.so.*.cxx"$std".abi 2>/dev/null | head -n 1) +mode=gate +if [ -z "$baseline" ]; then + mode=report + baseline=$(ls .abi-base/libjinja2cpp.so.*.cxx"$std".abi 2>/dev/null | head -n 1) +fi +summary=${GITHUB_STEP_SUMMARY:-/dev/null} + +if [ -z "$current" ]; then + echo "::error::no ABI dump for C++$std in .abi/" + exit 1 +fi +if [ -z "$baseline" ]; then + echo "No previous ABI dump for C++$std yet: nothing to compare." | tee -a "$summary" + exit 0 +fi + +abidiff --suppressions abi/suppressions.ini --no-show-locs "$baseline" "$current" > .abi/abidiff.txt 2>&1 +rc=$? +cat .abi/abidiff.txt +# Without the added symbols, any remaining change is a break: abidiff sets bit 8 only for +# removed symbols and vtable changes, and reports a changed layout (a new Settings field, a +# Value alternative, a widened member) with bit 4 alone +abidiff --suppressions abi/suppressions.ini --no-show-locs --no-added-syms "$baseline" "$current" > /dev/null 2>&1 +break_rc=$? + +soname() { grep -o -m 1 "soname='[^']*'" "$1" | cut -d"'" -f2; } +verdict="no ABI change" +if (( (rc | break_rc) & 3 )); then + verdict="abidiff failed (exit $rc, $break_rc)" +elif (( break_rc & 12 )); then + verdict="incompatible ABI change" +elif (( rc & 4 )); then + verdict="compatible ABI change (additions)" +fi + +{ + echo "### ABI, C++$std: $verdict" + echo + echo "Compared with \`$baseline\` ($mode mode)." + if (( rc != 0 )); then + echo + echo '
abidiff report' + echo + echo '```' + head -n 400 .abi/abidiff.txt + echo '```' + echo '
' + fi +} >> "$summary" + +if [ "$mode" = report ]; then + if (( (rc | break_rc) & 3 )); then + echo "::warning::$verdict; report mode does not fail" + elif (( break_rc & 12 )); then + echo "::notice::C++$std: $verdict against the last master build (report only until the 2.0 RC)" + fi + exit 0 +fi +if (( (rc | break_rc) & 3 )); then + echo "::error::$verdict" + exit 1 +fi +# A SOVERSION bump is only complete with its baseline: until the committed dump carries the +# new SONAME, every later break would pass as "SONAME differs" +if [ "$(soname "$baseline")" != "$(soname "$current")" ]; then + echo "::error::C++$std: the library's SONAME is $(soname "$current") but $baseline is for $(soname "$baseline"). Replace the baseline with the job's dump (artifact abi-cxx$std) in the same pull request as the SOVERSION bump (docs/tasks/0076)." + exit 1 +fi +if (( break_rc & 12 )); then + echo "::error::C++$std: $verdict without a SOVERSION bump. Bump SOVERSION in CMakeLists.txt and replace abi/ with the job's dump (artifact abi-cxx$std), or keep the ABI (docs/tasks/0076)." + exit 1 +fi +exit 0 diff --git a/scripts/gen_html_entities.py b/scripts/gen_html_entities.py index 713b747d0..8a5149bce 100644 --- a/scripts/gen_html_entities.py +++ b/scripts/gen_html_entities.py @@ -32,12 +32,12 @@ def main(): "// Generated by scripts/gen_html_entities.py from Python {}.{}'s html.entities.html5; do not edit.\n".format(*sys.version_info[:2]), "#ifndef JINJA2CPP_SRC_HTML_ENTITIES_H\n#define JINJA2CPP_SRC_HTML_ENTITIES_H\n", "#include \n#include \n#include \n#include \n", - "namespace jinja2::html\n{\n", + "namespace jinja2\n{\ninline namespace v2\n{\nnamespace html\n{\n", "// A named character reference and the one or two code points it stands for (0 when only one)\n" "struct HtmlEntity\n{\n const char* name;\n uint32_t first;\n uint32_t second;\n};\n", "// Sorted by name\nconstexpr HtmlEntity HtmlEntities[] = {\n" + "\n".join(rows) + "\n};\n", HELPER, - "} // namespace jinja2::html\n\n#endif // JINJA2CPP_SRC_HTML_ENTITIES_H\n", + "} // namespace html\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_HTML_ENTITIES_H\n", ] target = ROOT / "src" / "html_entities.h" text = "\n".join(parts) diff --git a/scripts/gen_unicode_case.py b/scripts/gen_unicode_case.py index 6cd74f592..572af3e6e 100644 --- a/scripts/gen_unicode_case.py +++ b/scripts/gen_unicode_case.py @@ -141,7 +141,7 @@ def main(): "// Generated by scripts/gen_unicode_case.py from Unicode {}; do not edit.\n".format(unicodedata.unidata_version), "#ifndef JINJA2CPP_SRC_UNICODE_CASE_H\n#define JINJA2CPP_SRC_UNICODE_CASE_H\n", '#include "unicode_tables.h"\n\n#include \n#include \n#include \n#include \n', - "namespace jinja2::unicode\n{\n", + "namespace jinja2\n{\ninline namespace v2\n{\nnamespace unicode\n{\n", "// cp + delta for every stride-th code point from first to last\n" "struct CaseRun\n{\n uint32_t first;\n uint32_t last;\n uint32_t stride;\n int32_t delta;\n};\n", "// A mapping to two or three code points; unused entries are 0\n" @@ -154,7 +154,7 @@ def main(): parts.append(range_table("Cased", "Cased and not Case_Ignorable", ranges(cased))) parts.append(range_table("CaseIgnorable", "Case_Ignorable (cased or not)", ranges(case_ignorable))) parts.append(HELPERS) - parts.append("} // namespace jinja2::unicode\n\n#endif // JINJA2CPP_SRC_UNICODE_CASE_H\n") + parts.append("} // namespace unicode\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_CASE_H\n") target = ROOT / "src" / "unicode_case.h" text = "\n".join(parts) # --check: CI (parity-expectations.yml) fails when the committed header is stale diff --git a/scripts/gen_unicode_printable.py b/scripts/gen_unicode_printable.py index 377c35d0e..6a9810d48 100644 --- a/scripts/gen_unicode_printable.py +++ b/scripts/gen_unicode_printable.py @@ -38,7 +38,7 @@ def generate(out): out.write("// Do not edit by hand.\n") out.write("#ifndef JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n#define JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n\n") out.write("#include \n#include \n#include \n\n") - out.write("namespace jinja2::detail\n{\n\n") + out.write("namespace jinja2\n{\ninline namespace v2\n{\nnamespace detail\n{\n\n") out.write("// True when Python's str.isprintable() holds for the code point, so repr() keeps it as is\n") out.write("inline bool IsPythonPrintable(uint32_t cp)\n{\n") out.write(" if (cp >= 0x20 && cp < 0x7f)\n {\n return true;\n }\n\n") @@ -51,7 +51,7 @@ def generate(out): " [](uint32_t val, const Range& r) { return val < r.first; });\n") out.write(" if (it == std::begin(nonPrintable))\n {\n return true;\n }\n") out.write(" --it;\n return cp > it->last;\n}\n\n") - out.write("} // namespace jinja2::detail\n\n#endif // JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n") + out.write("} // namespace detail\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n") def main(): diff --git a/scripts/gen_unicode_tables.py b/scripts/gen_unicode_tables.py index 9ca02b799..dd71049f9 100755 --- a/scripts/gen_unicode_tables.py +++ b/scripts/gen_unicode_tables.py @@ -37,7 +37,7 @@ def main(): "// Generated by scripts/gen_unicode_tables.py from Unicode {}; do not edit.\n".format(unicodedata.unidata_version), "#ifndef JINJA2CPP_SRC_UNICODE_TABLES_H\n#define JINJA2CPP_SRC_UNICODE_TABLES_H\n", "#include \n#include \n#include \n", - "namespace jinja2\n{\nnamespace unicode\n{\n", + "namespace jinja2\n{\ninline namespace v2\n{\nnamespace unicode\n{\n", "struct CodePointRange\n{\n uint32_t first;\n uint32_t last;\n};\n", table("WordRanges", "Python \\w above ASCII: str.isalnum()", ranges(lambda c: c.isalnum())), table("DecimalRanges", "Python \\d above ASCII: str.isdecimal()", ranges(lambda c: c.isdecimal())), @@ -73,7 +73,7 @@ def main(): return InRanges(SpaceRanges, cp); } """, - "} // namespace unicode\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_TABLES_H\n", + "} // namespace unicode\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_TABLES_H\n", ] target = ROOT / "src" / "unicode_tables.h" text = "\n".join(parts) diff --git a/src/binding/boost_json_parser.h b/src/binding/boost_json_parser.h index d60b41a58..bb12a5606 100644 --- a/src/binding/boost_json_parser.h +++ b/src/binding/boost_json_parser.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ template nonstd::expected Parse(std::basic_string_view json, boost::anys::unique_any& metadataJson) @@ -29,6 +31,7 @@ nonstd::expected Parse(std::basic_string_view json, b return Reflect(value); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_BOOST_JSON_PARSER_H diff --git a/src/binding/nlohmann_json_parser.h b/src/binding/nlohmann_json_parser.h index 50e65c6f0..9e9d5f012 100644 --- a/src/binding/nlohmann_json_parser.h +++ b/src/binding/nlohmann_json_parser.h @@ -9,6 +9,8 @@ namespace jinja2 { +inline namespace v2 +{ template nonstd::expected Parse(std::basic_string_view json, boost::anys::unique_any& metadataJson) @@ -26,6 +28,7 @@ nonstd::expected Parse(std::basic_string_view json, b } } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_NLOHMANN_JSON_PARSER_H diff --git a/src/binding/rapid_json_parser.h b/src/binding/rapid_json_parser.h index b0ba13634..6ddcb94de 100644 --- a/src/binding/rapid_json_parser.h +++ b/src/binding/rapid_json_parser.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -68,6 +70,7 @@ nonstd::expected Parse(std::basic_string_view json, b return Reflect(jsonDoc); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RAPID_JSON_PARSER_H diff --git a/src/cycle_collector.cpp b/src/cycle_collector.cpp index 8dfb8df3a..ea84ad81a 100644 --- a/src/cycle_collector.cpp +++ b/src/cycle_collector.cpp @@ -1,6 +1,7 @@ #include "cycle_collector.h" #include "internal_value.h" +#include "template_dict.h" #include #include @@ -12,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -21,7 +24,7 @@ struct Container { const ValueObject* object = nullptr; InternalValueList* list = nullptr; - InternalDict* dict = nullptr; + TemplateDict* dict = nullptr; }; Container FindContainer(const InternalValue& val) @@ -84,6 +87,11 @@ void ForEachItem(const Container& container, const Fn& fn) for (const auto& [key, item] : *container.dict) { fn(item); + // A key that is not a string keeps its first spelling, a tuple that can hold a container + if (container.dict->HasKeyValues()) + { + fn(container.dict->KeyOf(key)); + } } } } @@ -171,7 +179,7 @@ void CycleCollector::Collect() noexcept // The items of the containers found garbage: released after the walk, when every // container they keep alive has been emptied std::vector releasedLists; - std::vector releasedDicts; + std::vector releasedDicts; try { // Most of what a render noted is garbage by now with no cycle in it (a list built and @@ -252,7 +260,7 @@ void CycleCollector::Collect() noexcept else { releasedDicts.push_back(std::move(*node->container.dict)); - node->container.dict->clear(); + node->container.dict->Clear(); } } } @@ -262,4 +270,5 @@ void CycleCollector::Collect() noexcept m_notes->candidates.clear(); } +} // namespace v2 } // namespace jinja2 diff --git a/src/cycle_collector.h b/src/cycle_collector.h index 05c4b9e24..99cbb274e 100644 --- a/src/cycle_collector.h +++ b/src/cycle_collector.h @@ -10,6 +10,8 @@ namespace jinja2 { +inline namespace v2 +{ // Frees the reference cycles among the lists, dicts and namespaces a render builds // (docs/tasks/0110). They are shared by reference count, as Python's are, so a container @@ -78,6 +80,7 @@ class CycleCollector std::unique_ptr m_notes; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_CYCLE_COLLECTOR_H diff --git a/src/error_handling.h b/src/error_handling.h index 726a67e43..8eafae953 100644 --- a/src/error_handling.h +++ b/src/error_handling.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ struct ParseError { @@ -49,5 +51,6 @@ inline auto MakeParseError(ErrorCode code, Token tok, std::initializer_list namespace jinja2 { +inline namespace v2 +{ template void RenderErrorInfo(std::basic_string& result, const BasicErrorInfo& errInfo) @@ -330,4 +332,5 @@ std::wostream& operator<<(std::wostream& os, const ErrorInfoW& res) os << res.ToString(); return os; } +} // namespace v2 } // namespace jinja2 diff --git a/src/expression_evaluator.cpp b/src/expression_evaluator.cpp index e17293535..80c92b720 100644 --- a/src/expression_evaluator.cpp +++ b/src/expression_evaluator.cpp @@ -10,6 +10,7 @@ #include "python_format.h" #include "recursion_guard.h" #include "render_context.h" +#include "template_dict.h" #include "testers.h" #include "undefined.h" #include "value_methods.h" @@ -41,6 +42,8 @@ using namespace std::string_literals; namespace jinja2 { +inline namespace v2 +{ // Out of line, so that the paths without autoescape stay small void WriteEscaped(OutStream& stream, const InternalValue& val, IRendererCallback* callback) @@ -843,29 +846,6 @@ InternalValue TupleCreator::Evaluate(RenderContext& context) return list; } -namespace -{ -// Mapping keys are strings in the value model; integer and boolean keys are stored as -// their decimal or Python spelling, so {1: 'x'} has the key '1' -struct DictKeyGetter : public visitors::BaseVisitor -{ - using BaseVisitor::operator(); - - template - std::string operator()(const std::basic_string& str) const - { - return ConvertString(str); - } - template - std::string operator()(const std::basic_string_view& str) const - { - return ConvertString(str); - } - std::string operator()(int64_t val) const { return std::to_string(val); } - std::string operator()(bool val) const { return val ? "True" : "False"; } -}; -} // namespace - bool DictCreator::IsFoldable(const NodeArena& nodes) const { const auto items = nodes[m_exprs]; @@ -876,13 +856,15 @@ InternalValue DictCreator::Evaluate(RenderContext& context) { CheckStack(); const auto nodes = context.Nodes(); - InternalDict result; + TemplateDict result; for (const auto& item : nodes[m_exprs]) { // Python evaluates the key before the value; an assignment does not fix that order - auto key = Apply(nodes[item.key].Evaluate(context)); + const auto key = nodes[item.key].Evaluate(context); auto value = nodes[item.value].Evaluate(context); - result[std::move(key)] = std::move(value); + // Any hashable key; an unhashable one is a TypeError. A repeated key (1 and 1.0) keeps + // its first spelling and position and takes the last value + result.Set(key, std::move(value)); } return CreateMapAdapter(std::move(result)); @@ -1589,4 +1571,5 @@ ArenaCallParams ArenaCallParams::Make(NodeArena& nodes, const CallParamsInfo& pa result.kwParams = nodes.MakeSpan(kwParams); return result; } +} // namespace v2 } // namespace jinja2 diff --git a/src/expression_evaluator.h b/src/expression_evaluator.h index 06cf78ecb..dfa99c9a3 100644 --- a/src/expression_evaluator.h +++ b/src/expression_evaluator.h @@ -33,6 +33,8 @@ namespace jinja2 { +inline namespace v2 +{ class ExpressionEvaluatorBase : public ArenaNode { @@ -1064,6 +1066,7 @@ ParsedArgumentsInfo ParseCallParamsInfo(const std::vector& args, c CallParams EvaluateCallParams(const CallParamsInfo& info, RenderContext& context); CallParams EvaluateCallParams(const ArenaCallParams& info, RenderContext& context); } // namespace helpers +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_EXPRESSION_EVALUATOR_H diff --git a/src/expression_parser.cpp b/src/expression_parser.cpp index b5aba9a65..2c98d3afd 100644 --- a/src/expression_parser.cpp +++ b/src/expression_parser.cpp @@ -28,6 +28,8 @@ namespace jinja2 { +inline namespace v2 +{ ParseError::ParseError(const ParseError&) = default; ParseError::ParseError(ParseError&& other) noexcept @@ -1281,4 +1283,5 @@ ExpressionParser::ParseResult> ExpressionParser::ParseIfEx return result; } +} // namespace v2 } // namespace jinja2 diff --git a/src/expression_parser.h b/src/expression_parser.h index b4687a373..1f1504437 100644 --- a/src/expression_parser.h +++ b/src/expression_parser.h @@ -20,6 +20,8 @@ namespace jinja2 { +inline namespace v2 +{ // Jinja2 rejects a filter or test it does not know when it compiles the template, except in // an `if` statement or an inline if, which leave the check to the call (Frame.soft_frame in // jinja2/compiler.py). The expression parsers of one template report each unknown name here. @@ -211,6 +213,7 @@ class ExpressionParser unsigned m_operators = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_EXPRESSION_PARSER_H diff --git a/src/filesystem_handler.cpp b/src/filesystem_handler.cpp index c480524b4..17759ff3f 100644 --- a/src/filesystem_handler.cpp +++ b/src/filesystem_handler.cpp @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ using TargetFileStream = std::variant; @@ -277,4 +279,5 @@ bool RealFileSystem::IsEqual(const IComparable& other) const return m_rootFolder == ptr->m_rootFolder; } +} // namespace v2 } // namespace jinja2 diff --git a/src/filter_factories.cpp b/src/filter_factories.cpp index 884960444..7342550e8 100644 --- a/src/filter_factories.cpp +++ b/src/filter_factories.cpp @@ -22,6 +22,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -163,4 +165,5 @@ FilterPtr CreateFilter(const std::string& filterName, const CallParamsInfo& para return std::make_shared(filterName, params, std::move(callable)); } +} // namespace v2 } // namespace jinja2 diff --git a/src/filters.cpp b/src/filters.cpp index 5867861eb..f1f3304cb 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -8,6 +8,7 @@ #include "out_stream.h" // IWYU pragma: keep (GetStreamOnString returns an OutStream by value) #include "render_context.h" #include "string_builder.h" +#include "template_dict.h" #include "testers.h" #include "undefined.h" #include "unicode_tables.h" @@ -41,7 +42,11 @@ using namespace std::string_literals; -namespace jinja2::filters +namespace jinja2 +{ +inline namespace v2 +{ +namespace filters { // Which values Python can order against each other with `<` @@ -467,21 +472,37 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont struct SortItem { KeyValuePair* pair; + // The lowered key when case-insensitive and every key is a string std::string sortKey; }; bool (*comparator)(const SortItem& left, const SortItem& right) = nullptr; bool lowerKeys = false; + bool sortByKey = false; + bool keysAreStrings = true; + + auto entries = map->GetEntries(); if (AsString(byVal) == "key") // Sort by key { + sortByKey = true; lowerKeys = !ConvertToBool(isCsVal); - if (lowerKeys) + // Narrow text, as map keys are UTF-8 (a template dict converts wide keys) + keysAreStrings = std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return AsStringView(entry.key).has_value(); }); + if (keysAreStrings && lowerKeys) { comparator = [](const SortItem& left, const SortItem& right) { return left.sortKey < right.sortKey; }; } + else if (keysAreStrings) + { + comparator = [](const SortItem& left, const SortItem& right) { return *AsStringView(left.pair->key) < *AsStringView(right.pair->key); }; + } else { - comparator = [](const SortItem& left, const SortItem& right) { return left.pair->key < right.pair->key; }; + // Python lowers only str keys, and a str key next to a key of another type + // fails the comparison either way, so other keys compare as they are + comparator = [](const SortItem& left, const SortItem& right) { + return CompareForOrder(left.pair->key, right.pair->key, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive); + }; } } else if (AsString(byVal) == "value") @@ -505,15 +526,14 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont throw std::runtime_error(R"(You can only sort by either "key" or "value")"); } - auto entries = map->GetEntries(); std::vector tempVector; tempVector.reserve(entries.size()); for (auto& entry : entries) { SortItem item{ &entry, std::string() }; - if (lowerKeys) + if (sortByKey && keysAreStrings && lowerKeys) { - detail::LowerUtf8(entry.key, item.sortKey); + detail::LowerUtf8(*AsStringView(entry.key), item.sortKey); } tempVector.push_back(std::move(item)); } @@ -956,37 +976,6 @@ struct UniqueItem int64_t idx; }; -// The type Python's hash() rejects in a value, if any: a list or a dict, also inside a -// tuple. A range hashes, and so does an object stored as a map (namespace(), cycler()) or -// a host object (a reflected struct, which Python would hold as an object, or a JSON object) -struct UnhashableTypeGetter : visitors::BaseVisitor -{ - using visitors::BaseVisitor::operator(); - - const char* operator()(const ListRef& list) const - { - if (list.GetRangeInfo()) - { - return nullptr; - } - if (!list.IsTuple()) - { - return "list"; - } - const char* result = nullptr; - list.ForEach([&result](const InternalValue& item) { - result = Apply(item); - return result == nullptr; - }); - return result; - } - const char* operator()(const MapRef& map) const - { - return map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst ? nullptr : "dict"; - } - const char* operator()(const KeyValuePair& pair) const { return Apply(pair.value); } -}; - // Drops repeated keys, keeping the first of each in its original position void DropDuplicates(std::vector& items, BinaryExpression::CompareType compType) { @@ -1047,7 +1036,7 @@ InternalValue Unique(const ListAdapter& list, // Jinja2 keeps a set of the keys it saw for (const auto& item : items) { - if (const char* type = Apply(item.val)) + if (const char* type = UnhashableType(item.val)) { throw std::runtime_error("unhashable type: '"s + type + "'"); } @@ -1680,7 +1669,7 @@ struct ValueConverterImpl : visitors::BaseVisitor<> return InternalValue(); } - return ListAdapter::CreateAdapter(InternalValueList{ InternalValue(val.key), val.value }); + return ListAdapter::CreateAdapter(InternalValueList{ val.key, val.value }); } InternalValue operator()(const MapRef& val) const @@ -1694,9 +1683,9 @@ struct ValueConverterImpl : visitors::BaseVisitor<> throw std::runtime_error("'"s + objectClass + "' object is not iterable"); } - auto keys = val.GetKeys(); + auto keys = val.GetKeyValues(); auto numKeys = keys.size(); - return ListAdapter::CreateAdapter(numKeys, [values = std::move(keys)](size_t idx) { return InternalValue(values[idx]); }); + return ListAdapter::CreateAdapter(numKeys, [values = std::move(keys)](size_t idx) { return values[idx]; }); } template @@ -2350,7 +2339,7 @@ InternalValue ValueConverter::Items(const InternalValue& baseVal, RenderContext& InternalValueList items; for (auto& [key, value] : map->GetEntries()) { - items.emplace_back(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(std::move(key)), std::move(value) }).MarkAsTuple()); + items.emplace_back(ListAdapter::CreateAdapter(InternalValueList{ std::move(key), std::move(value) }).MarkAsTuple()); } InternalValue result = ListAdapter::CreateAdapter(std::move(items)); if (baseVal.ShouldExtendLifetime()) @@ -2604,4 +2593,6 @@ InternalValue UserDefinedFilter::Filter(const InternalValue& baseVal, RenderCont return callable->GetExpressionCallable()(callParams, context); } -} // namespace jinja2::filters +} // namespace filters +} // namespace v2 +} // namespace jinja2 diff --git a/src/filters.h b/src/filters.h index 6921b9c1f..a5a2bf407 100644 --- a/src/filters.h +++ b/src/filters.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ using FilterPtr = std::shared_ptr; using FilterParams = CallParamsInfo; @@ -339,6 +341,7 @@ class UserDefinedFilter : public FilterBase }; } // namespace filters +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_FILTERS_H diff --git a/src/function_base.h b/src/function_base.h index 2af06a502..f0bc1d502 100644 --- a/src/function_base.h +++ b/src/function_base.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ // The parameters of a filter or tester kind, built once (a function-local static): an instance // keeps a pointer to its table, so the names and defaults are stored once per kind using ArgumentsTable = std::vector; @@ -166,6 +168,7 @@ inline bool FunctionBase::ParseParams(const ArgumentsTable& argsInfo, const Call return result; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_FUNCTION_BASE_H diff --git a/src/generic_adapters.h b/src/generic_adapters.h index a7b982ffd..95313c163 100644 --- a/src/generic_adapters.h +++ b/src/generic_adapters.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ template class IndexedEnumeratorImpl : public Base @@ -331,6 +333,7 @@ inline std::optional IndexedListItemAccessorImpl::CreateEn return MakeEnumerator(this); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_GENERIC_ADAPTERS_H diff --git a/src/generic_list.cpp b/src/generic_list.cpp index cfb5b06c2..2652c907e 100644 --- a/src/generic_list.cpp +++ b/src/generic_list.cpp @@ -3,6 +3,8 @@ namespace jinja2 { +inline namespace v2 +{ GenericList::iterator GenericList::begin() const { @@ -47,4 +49,5 @@ bool operator!=(const GenericList& lhs, const GenericList& rhs) return !(lhs == rhs); } +} // namespace v2 } // namespace jinja2 diff --git a/src/global_functions.cpp b/src/global_functions.cpp index 573a1a83c..445d01aa0 100644 --- a/src/global_functions.cpp +++ b/src/global_functions.cpp @@ -2,6 +2,7 @@ #include "internal_value.h" #include "markup.h" #include "python_format.h" +#include "template_dict.h" #include "undefined.h" #include "value_visitors.h" @@ -28,6 +29,8 @@ using namespace std::string_literals; namespace jinja2 { +inline namespace v2 +{ namespace { // jinja2.constants.LOREM_IPSUM_WORDS @@ -78,19 +81,6 @@ ParsedArguments ParseArgs(const std::initializer_list& argsInfo, c return args; } -std::string KeyToString(const InternalValue& key) -{ - if (IsStringValue(key)) - { - return AsString(key); - } - - // Mapping keys are strings (task 0036): store other keys by their printed form - std::string result; - Apply>(key, result); - return result; -} - // Python's range and randrange take integers (bool among them) and nothing else; a plain // undefined is a missing argument, left to the caller void CheckIntegerArgument(const InternalValue& arg) @@ -139,9 +129,31 @@ InternalValue CallRange(const CallParams& params, RenderContext&) return ListAdapter::CreateRange(start, stop, step); } +// One item of dict(pairs) into `result` +void CollectDictPair(const InternalValue& item, TemplateDict& result) +{ + if (const auto* pair = GetIf(&item)) + { + result.Set(pair->key, pair->value); + return; + } + // Any iterable of two items is a pair, a two-character string included + bool isConverted = false; + auto itemList = ConvertToList(item, isConverted, false); + if (!isConverted) + { + throw std::runtime_error("cannot convert dictionary update sequence element to a sequence"); + } + auto pair = itemList.ToValueList(); + if (pair.size() != 2) + { + throw std::runtime_error("dictionary update sequence element has wrong length; 2 is required"); + } + result.Set(pair[0], pair[1]); +} + // The items of dict(mapping_or_pairs, **kwargs), into `result` -template -void CollectDictItems(const CallParams& params, const char* fnName, Map& result) +void CollectDictItems(const CallParams& params, const char* fnName, TemplateDict& result) { if (params.posParams.size() > 1) { @@ -159,33 +171,16 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) } if (map) { - for (auto& key : map->GetKeys()) + for (auto& [key, value] : map->GetEntries()) { - result[key] = map->GetValueByName(key); + result.Set(key, std::move(value)); } } else if (auto list = AsList(source)) { for (const auto& item : *list) { - if (const auto* pair = GetIf(&item)) - { - result[pair->key] = pair->value; - continue; - } - // Any iterable of two items is a pair, a two-character string included - bool isConverted = false; - auto itemList = ConvertToList(item, isConverted, false); - if (!isConverted) - { - throw std::runtime_error("cannot convert dictionary update sequence element to a sequence"); - } - auto pair = itemList.ToValueList(); - if (pair.size() != 2) - { - throw std::runtime_error("dictionary update sequence element has wrong length; 2 is required"); - } - result[KeyToString(pair[0])] = pair[1]; + CollectDictPair(item, result); } } else if (!IsEmpty(source)) @@ -196,14 +191,14 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) for (const auto& [name, value] : params.kwParams) { - result[name] = value; + result.SetString(name, value); } } // dict(mapping_or_pairs, **kwargs) InternalValue CallDict(const CallParams& params, RenderContext&) { - InternalDict result; + TemplateDict result; CollectDictItems(params, "dict", result); return CreateMapAdapter(std::move(result)); } @@ -211,7 +206,7 @@ InternalValue CallDict(const CallParams& params, RenderContext&) // namespace(mapping_or_pairs, **kwargs): takes its arguments as dict() does InternalValue CallNamespace(const CallParams& params, RenderContext&) { - InternalDict result; + TemplateDict result; CollectDictItems(params, "namespace", result); return CreateNamespaceAdapter(std::move(result)); } @@ -530,4 +525,5 @@ const InternalValueMap& GetBuiltinGlobals(bool withI18n) }(); return withI18n ? globalsWithI18n : globals; } +} // namespace v2 } // namespace jinja2 diff --git a/src/helpers.h b/src/helpers.h index b189b84c9..ec0eab7f1 100644 --- a/src/helpers.h +++ b/src/helpers.h @@ -9,6 +9,8 @@ namespace jinja2 { +inline namespace v2 +{ struct MultiStringLiteral { const char* charValue; @@ -281,6 +283,7 @@ Sequence CompileEscapes(Sequence s, bool* ok = nullptr) return s; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_HELPERS_H diff --git a/src/html_entities.h b/src/html_entities.h index ffe3f2de9..b02240303 100644 --- a/src/html_entities.h +++ b/src/html_entities.h @@ -8,7 +8,11 @@ #include #include -namespace jinja2::html +namespace jinja2 +{ +inline namespace v2 +{ +namespace html { // A named character reference and the one or two code points it stands for (0 when only one) @@ -2261,6 +2265,8 @@ inline const HtmlEntity* FindHtmlEntity(std::string_view name) return it != std::end(HtmlEntities) && std::string_view(it->name) == name ? &*it : nullptr; } -} // namespace jinja2::html +} // namespace html +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_HTML_ENTITIES_H diff --git a/src/internal_value.cpp b/src/internal_value.cpp index 8f4405d97..163585ee0 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -2,6 +2,7 @@ #include "expression_evaluator.h" #include "generic_adapters.h" +#include "template_dict.h" #include "undefined.h" #include "value_visitors.h" @@ -31,6 +32,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -435,7 +438,28 @@ struct SubscriptionVisitor : public visitors::BaseVisitor<> return values.GetValueByIndex(index); } - InternalValue operator()(const MapRef& /*values*/, int64_t /*index*/) const { return InternalValue(); } + // A key that is not a string is a key of a dict the template owns (1, 1.5, True, none, a tuple); + // other maps hold strings only, and an unhashable key reads as undefined, as Jinja2 swallows + // the TypeError + [[nodiscard]] static InternalValue FindKey(const MapRef& values, const InternalValue& key) + { + auto found = values.FindItem(key); + return found ? std::move(*found) : InternalValue(); + } + + InternalValue operator()(const MapRef& values, int64_t key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, bool key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, double key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, NoneValue /*key*/) const { return FindKey(values, InternalValue(NoneValue{})); } + InternalValue operator()(const MapRef& values, const KeyValuePair& key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, const ListRef& key) const + { + if (!key.IsTuple() && !key.GetRangeInfo()) + { + return InternalValue(); + } + return FindKey(values, InternalValue(key.ToAdapter())); + } // A dictsort pair reads as the 2-tuple (key, value) InternalValue operator()(const KeyValuePair& pair, int64_t index) const @@ -891,9 +915,9 @@ struct ListConverter : public visitors::BaseVisitor> } InternalValueList list; - for (auto& k : map.GetKeys()) + for (auto& k : map.GetKeyValues()) { - list.emplace_back(TargetString(k)); + list.emplace_back(std::move(k)); } return ListAdapter::CreateAdapter(std::move(list)); @@ -1668,12 +1692,39 @@ std::vector IMapAccessor::GetEntries() const for (auto& key : keys) { auto value = GetItem(key); - result.push_back(KeyValuePair{ std::move(key), std::move(value) }); + result.push_back(KeyValuePair{ InternalValue(key), std::move(value) }); + } + + return result; +} + +InternalValueList IMapAccessor::GetKeyValues() const +{ + InternalValueList result; + auto keys = GetKeys(); + result.reserve(keys.size()); + for (auto& key : keys) + { + result.emplace_back(key); } return result; } +std::optional IMapAccessor::FindItem(const InternalValue& key) const +{ + if (IsStringValue(key)) + { + const auto name = AsString(key); + if (HasValue(name)) + { + return GetItem(name); + } + } + + return std::nullopt; +} + template class Holder, bool canModify, typename Map = InternalValueMap> class InternalValueMapAdapter : public MapAccessorImpl> { @@ -1716,7 +1767,7 @@ class InternalValueMapAdapter : public MapAccessorImpl m_values; }; -// A dict the template owns (dict literals, kwargs, dict()): shared like a Python dict -class SharedDictAdapter : public InternalValueMapAdapter +// A dict the template owns (dict literals, kwargs, dict()): shared like a Python dict, and keyed +// like one (TemplateDict). A string key costs one lookup as in any other map +class SharedDictAdapter : public MapAccessorImpl { public: - using InternalValueMapAdapter::InternalValueMapAdapter; + explicit SharedDictAdapter(TemplateDict&& values) + : m_values(std::move(values)) + { + } + [[nodiscard]] size_t GetSize() const override { return m_values.Get().size(); } + [[nodiscard]] bool HasValue(const std::string& name) const override { return m_values.Get().FindString(name) != m_values.Get().end(); } + [[nodiscard]] InternalValue GetItem(const std::string& name) const override + { + auto& vals = m_values.Get(); + auto p = vals.FindString(name); + return p == vals.end() ? InternalValue() : p->second; + } + // The keys spelled as str(), so that a reader of the string keyed view never sees a token + [[nodiscard]] std::vector GetKeys() const override + { + std::vector result; + auto& vals = m_values.Get(); + result.reserve(vals.size()); + for (const auto& [stored, value] : vals) + { + result.push_back(DictKeyText(vals.KeyOf(stored))); + } + + return result; + } + [[nodiscard]] InternalValueList GetKeyValues() const override + { + InternalValueList result; + auto& vals = m_values.Get(); + result.reserve(vals.size()); + for (const auto& [stored, value] : vals) + { + result.push_back(vals.KeyOf(stored)); + } + + return result; + } + [[nodiscard]] std::vector GetEntries() const override + { + std::vector result; + auto& vals = m_values.Get(); + result.reserve(vals.size()); + for (const auto& [stored, value] : vals) + { + result.push_back(KeyValuePair{ vals.KeyOf(stored), value }); + } + + return result; + } + [[nodiscard]] std::optional FindItem(const InternalValue& key) const override + { + auto& vals = m_values.Get(); + auto p = vals.Find(key); + if (p == vals.end()) + { + return std::nullopt; + } + return p->second; + } + + bool SetValue(std::string name, const InternalValue& val) const override + { + m_values.Get().SetString(std::move(name), val); + return true; + } + [[nodiscard]] bool ShouldExtendLifetime() const override { return m_values.ShouldExtendLifetime(); } + [[nodiscard]] bool IsEqual(const IComparable& other) const override + { + const auto* val = dynamic_cast(&other); + if (!val) + { + return false; + } + return m_values == val->m_values; + } [[nodiscard]] const void* GetIdentity() const override { return &m_values.Get(); } - [[nodiscard]] InternalDict* GetMutableItems() const override { return &m_values.Get(); } + [[nodiscard]] TemplateDict* GetMutableItems() const override { return &m_values.Get(); } [[nodiscard]] bool OwnsMutableItemsAlone() const override { return m_values.IsUnique(); } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const override { return MapAttrPolicy::MethodsFirst; } [[nodiscard]] GenericMap CreateGenericMap() const override { + if (auto snapshot = CreateSnapshot()) + { + return std::move(*snapshot); + } return GenericMap([accessor = *this]() -> const IMapItemAccessor* { return &accessor; }); } + +protected: + // The public map has string keys only (D13: docs/release-2.0.md, docs/tasks/0036). A dict + // with a key that is not a string is handed over as a copy, not a live view, with the keys + // spelled as str() does: {1: 'x', None: 'y'} has the keys "1" and "None". Keys that spell + // alike ({1: 'x', '1': 'y'}) collapse, and the later value wins at the position of the first, + // as Python's {str(k): v for k, v in d.items()} has it. A dict of string keys stays a live view + [[nodiscard]] std::optional CreateSnapshot() const + { + auto& vals = m_values.Get(); + if (!vals.HasKeyValues()) + { + return std::nullopt; + } + ValuesMap snapshot; + for (const auto& [stored, value] : vals) + { + snapshot[DictKeyText(vals.KeyOf(stored))] = IntValue2Value(value); + } + return CreateMapAdapter(std::move(snapshot)).CreateGenericMap(); + } + + // Mutable for SetValue, the one writer (IMapAccessor::SetValue) + mutable BySharedMutable m_values; }; // namespace(): shared like a dict, but an object with attributes rather than a dict to the @@ -1773,11 +1927,20 @@ class NamespaceAdapter : public SharedDictAdapter public: using SharedDictAdapter::SharedDictAdapter; + // Its attributes are names: ns[1] is undefined, as in Python + [[nodiscard]] std::optional FindItem(const InternalValue& key) const override + { + return IsStringValue(key) ? SharedDictAdapter::FindItem(key) : std::nullopt; + } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const override { return MapAttrPolicy::KeysOnly; } [[nodiscard]] bool IsNamespace() const override { return true; } [[nodiscard]] const char* GetObjectClass() const override { return "Namespace"; } [[nodiscard]] GenericMap CreateGenericMap() const override { + if (auto snapshot = CreateSnapshot()) + { + return std::move(*snapshot); + } return GenericMap([accessor = *this]() -> const IMapItemAccessor* { return &accessor; }); } }; @@ -1868,6 +2031,21 @@ class GenericMapAdapter : public MapAccessorImpl> Holder m_values; }; +// A key of user data as LendItem lends its string items: a view while the data outlives the +// render, so reading entries copies no key text (Render/dict_ops) +template class Holder, typename T> +InternalValue LendKey([[maybe_unused]] const Holder& holder, const std::string& key) +{ + if constexpr (std::is_same_v, ByRef>) + { + return InternalValue(TargetStringView(std::string_view(key))); + } + else + { + return InternalValue(key); + } +} + template class Holder> class ValuesMapAdapter : public MapAccessorImpl> { @@ -1910,7 +2088,7 @@ class ValuesMapAdapter : public MapAccessorImpl> for (const auto& [key, value] : m_values.Get()) { - result.push_back(KeyValuePair{ key, LendItem(m_values, value) }); + result.push_back(KeyValuePair{ LendKey(m_values, key), LendItem(m_values, value) }); } return result; @@ -1993,15 +2171,25 @@ MapAdapter CreateMapAdapter(InternalValueMap&& values) } MapAdapter CreateMapAdapter(InternalDict&& values) +{ + return MapAdapter(MakeRef(TemplateDict(std::move(values)))); +} + +MapAdapter CreateMapAdapter(TemplateDict&& values) { return MapAdapter(MakeRef(std::move(values))); } -MapAdapter CreateNamespaceAdapter(InternalDict&& values) +MapAdapter CreateNamespaceAdapter(TemplateDict&& values) { return MapAdapter(MakeRef(std::move(values))); } +MapAdapter CreateNamespaceAdapter(InternalDict&& values) +{ + return MapAdapter(MakeRef(TemplateDict(std::move(values)))); +} + MapAdapter CreateObjectAdapter(InternalValueMap&& values, const char* className) { return MapAdapter(MakeRef(std::move(values), className)); @@ -2051,7 +2239,7 @@ struct OutputValueConvertor result_t operator()(const MapRef& adapter) const { return result_t(adapter.CreateGenericMap()); } result_t operator()(const ListRef& adapter) const { return result_t(adapter.CreateGenericList()); } result_t operator()(const ValueRef& ref) const { return ref.get(); } - result_t operator()(const KeyValuePair& pair) const { return ValuesMap{ { "key", Value(pair.key) }, { "value", IntValue2Value(pair.value) } }; } + result_t operator()(const KeyValuePair& pair) const { return ValuesMap{ { "key", IntValue2Value(pair.key) }, { "value", IntValue2Value(pair.value) } }; } result_t operator()(const Callable&) const { return result_t(); } result_t operator()(const UserCallable&) const { return result_t(); } result_t operator()(const std::shared_ptr&) const { return result_t(); } @@ -2246,4 +2434,5 @@ bool Callable::IsSame(const Callable& other) const return left != nullptr && right != nullptr && left->val.IsEqual(right->val); } +} // namespace v2 } // namespace jinja2 diff --git a/src/internal_value.h b/src/internal_value.h index c27717ae7..7db7b5a70 100644 --- a/src/internal_value.h +++ b/src/internal_value.h @@ -45,6 +45,8 @@ namespace jinja2 { +inline namespace v2 +{ template class ReferenceWrapper @@ -414,6 +416,9 @@ void ReserveHint(List& list, size_t size) // dicts do; scopes and other lookup-only maps stay InternalValueMap (docs/tasks/0031) using InternalDict = OrderedMap; +// A dict with Python's keys: any hashable value, over the string keys of InternalDict (template_dict.h) +class TemplateDict; + struct IListAccessor; struct IListAccessorEnumerator; @@ -540,6 +545,11 @@ struct IMapAccessor : ValueObject // The pairs in GetKeys() order. The default looks every key up again; adapters that // own a map read its entries directly [[nodiscard]] virtual std::vector GetEntries() const; + // The keys as template values, in GetKeys() order. The default turns each string into a value + [[nodiscard]] virtual InternalValueList GetKeyValues() const; + // The value under `key`, if there is one. The default answers a string key through + // HasValue/GetItem and has no other key, as host maps hold only strings + [[nodiscard]] virtual std::optional FindItem(const InternalValue& key) const; // Const as reading is, so that a value can hold its accessor as a const object; the // accessor changes its own mutable state (docs/tasks/0140). By value: overrides store // the key. `set ns.attr = ...` has no use for the result. @@ -553,7 +563,7 @@ struct IMapAccessor : ValueObject // See IListAccessor::GetIdentity [[nodiscard]] virtual const void* GetIdentity() const { return this; } // See IListAccessor::GetMutableItems - [[nodiscard]] virtual InternalDict* GetMutableItems() const { return nullptr; } + [[nodiscard]] virtual TemplateDict* GetMutableItems() const { return nullptr; } // See IListAccessor::OwnsMutableItemsAlone [[nodiscard]] virtual bool OwnsMutableItemsAlone() const { return false; } // How x.name resolves against Python's dict methods (items, get, ...) @@ -754,9 +764,10 @@ class MapRef [[nodiscard]] InternalValue GetValueByName(const std::string& name) const; [[nodiscard]] const void* GetIdentity() const { return m_accessor ? m_accessor->GetIdentity() : nullptr; } [[nodiscard]] bool HasAttributes() const { return m_accessor != nullptr && m_accessor->HasAttributes(); } - [[nodiscard]] std::vector GetKeys() const { return m_accessor ? m_accessor->GetKeys() : std::vector(); } [[nodiscard]] std::vector GetEntries() const; - [[nodiscard]] InternalDict* GetMutableItems() const { return m_accessor ? m_accessor->GetMutableItems() : nullptr; } + [[nodiscard]] InternalValueList GetKeyValues() const; + [[nodiscard]] std::optional FindItem(const InternalValue& key) const; + [[nodiscard]] TemplateDict* GetMutableItems() const { return m_accessor ? m_accessor->GetMutableItems() : nullptr; } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const { return m_accessor ? m_accessor->GetAttrPolicy() : MapAttrPolicy::KeysOnly; } [[nodiscard]] bool IsNamespace() const { return m_accessor != nullptr && m_accessor->IsNamespace(); } [[nodiscard]] const char* GetObjectClass() const { return m_accessor ? m_accessor->GetObjectClass() : nullptr; } @@ -804,9 +815,10 @@ class MapAdapter [[nodiscard]] InternalValue GetValueByName(const std::string& name) const; [[nodiscard]] const void* GetIdentity() const { return Ref().GetIdentity(); } [[nodiscard]] bool HasAttributes() const { return Ref().HasAttributes(); } - [[nodiscard]] std::vector GetKeys() const { return Ref().GetKeys(); } [[nodiscard]] std::vector GetEntries() const; - [[nodiscard]] InternalDict* GetMutableItems() const { return Ref().GetMutableItems(); } + [[nodiscard]] InternalValueList GetKeyValues() const; + [[nodiscard]] std::optional FindItem(const InternalValue& key) const; + [[nodiscard]] TemplateDict* GetMutableItems() const { return Ref().GetMutableItems(); } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const { return Ref().GetAttrPolicy(); } [[nodiscard]] bool IsNamespace() const { return Ref().IsNamespace(); } [[nodiscard]] const char* GetObjectClass() const { return Ref().GetObjectClass(); } @@ -1355,8 +1367,10 @@ using InternalValueMap = robin_hood::unordered_map MapAdapter::GetEntries() const return Ref().GetEntries(); } +inline InternalValueList MapRef::GetKeyValues() const +{ + return m_accessor ? m_accessor->GetKeyValues() : InternalValueList(); +} + +inline InternalValueList MapAdapter::GetKeyValues() const +{ + return Ref().GetKeyValues(); +} + +inline std::optional MapRef::FindItem(const InternalValue& key) const +{ + return m_accessor ? m_accessor->FindItem(key) : std::nullopt; +} + +inline std::optional MapAdapter::FindItem(const InternalValue& key) const +{ + return Ref().FindItem(key); +} + // The list or map a value holds, if it holds one: `if (auto list = AsList(val)) list->GetSize()`. // Not for temporaries, which the view would outlive inline std::optional AsList(const InternalValue& val) @@ -1746,6 +1780,7 @@ static_assert(std::is_nothrow_move_assignable_v); Value IntValue2Value(const InternalValue& val); Value OptIntValue2Value(std::optional val); +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_INTERNAL_VALUE_H diff --git a/src/lexer.cpp b/src/lexer.cpp index 7a33af003..4af410a48 100644 --- a/src/lexer.cpp +++ b/src/lexer.cpp @@ -13,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -77,4 +79,5 @@ InternalValue ParseNumberLiteral(std::string number) return InternalValue(std::strtod(digits, nullptr)); } +} // namespace v2 } // namespace jinja2 diff --git a/src/lexer.h b/src/lexer.h index f932d0ce0..2596b35ae 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ struct CharRange { size_t startOffset; @@ -919,6 +921,7 @@ class LexScanner } }; +} // namespace v2 } // namespace jinja2 namespace std diff --git a/src/load_settings.h b/src/load_settings.h index 9ce584d18..684bf0dd5 100644 --- a/src/load_settings.h +++ b/src/load_settings.h @@ -11,7 +11,11 @@ #include #include -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // Delimiters and line prefixes from Settings, in the character type of a template @@ -122,6 +126,8 @@ using LoadSettingsPtr = std::shared_ptr; // For templates made without an environment const LoadSettingsPtr& DefaultLoadSettings(); -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_LOAD_SETTINGS_H diff --git a/src/lookup_result.h b/src/lookup_result.h index 8107208c5..3586d255d 100644 --- a/src/lookup_result.h +++ b/src/lookup_result.h @@ -7,6 +7,8 @@ namespace jinja2 { +inline namespace v2 +{ // Where a lookup found a value, or nothing: a nullable reference to a value stored elsewhere // (a scope, a constant node), read in place instead of copied (docs/tasks/0088, 0117). It is // valid while what holds the value is unchanged: for a variable, until a scope of the @@ -47,6 +49,7 @@ class [[nodiscard]] BasicLookupResult using LookupResult = BasicLookupResult; // A variable found for changing in place (docs/tasks/0020) using MutableLookupResult = BasicLookupResult; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_LOOKUP_RESULT_H diff --git a/src/loop_attr.h b/src/loop_attr.h index 30de75c8f..d77d45f2c 100644 --- a/src/loop_attr.h +++ b/src/loop_attr.h @@ -6,6 +6,8 @@ namespace jinja2 { +inline namespace v2 +{ // An attribute of a for loop's `loop` object. `loop.` in the template is resolved to // one of these at Load, so a render reads the attribute without comparing strings // (docs/design/perf-design-overview.md row 17; 0117 P1) @@ -61,6 +63,7 @@ inline LoopAttr FindLoopAttr(std::string_view name) } return LoopAttr::None; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_LOOP_ATTR_H diff --git a/src/make_unexpected.h b/src/make_unexpected.h index 475394766..821f760a7 100644 --- a/src/make_unexpected.h +++ b/src/make_unexpected.h @@ -7,6 +7,8 @@ namespace jinja2 { +inline namespace v2 +{ // The one place that names the backing `expected` library's error factory, so that // switching it (docs/api-2.0.md, decision 4) touches this function and not every @@ -17,6 +19,7 @@ auto MakeUnexpected(E&& error) return nonstd::make_unexpected(std::forward(error)); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_MAKE_UNEXPECTED_H diff --git a/src/markup.h b/src/markup.h index e5f34f6e9..bfb702180 100644 --- a/src/markup.h +++ b/src/markup.h @@ -16,6 +16,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -218,6 +220,7 @@ inline void WriteOutput(OutStream& stream, const InternalValue& val, RenderConte WriteEscaped(stream, val, context.GetRendererCallback()); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_MARKUP_H diff --git a/src/name_resolver.cpp b/src/name_resolver.cpp index e88a19a50..0d295c0b4 100644 --- a/src/name_resolver.cpp +++ b/src/name_resolver.cpp @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ NameResolver::FrameId NameResolver::Push(FrameId parent, Kind kind) { const auto id = static_cast(m_frames.size()); @@ -618,4 +620,5 @@ std::vector NameResolver::UndeclaredNames(const NodeArena& nod } return names; } +} // namespace v2 } // namespace jinja2 diff --git a/src/name_resolver.h b/src/name_resolver.h index 0ebac158b..5aa9fff48 100644 --- a/src/name_resolver.h +++ b/src/name_resolver.h @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ class BindUndefinedStatement; class ForStatement; class IRendererBase; @@ -277,6 +279,7 @@ class NameResolver bool m_inBranch = false; bool m_hasSlots = false; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_NAME_RESOLVER_H diff --git a/src/node_arena.cpp b/src/node_arena.cpp index f5cc922da..d039dd8dd 100644 --- a/src/node_arena.cpp +++ b/src/node_arena.cpp @@ -20,6 +20,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { void OffsetList::Grow() @@ -259,4 +261,5 @@ SealedArena NodeArena::SealWith(boost::span roots) } return sealed; } +} // namespace v2 } // namespace jinja2 diff --git a/src/node_arena.h b/src/node_arena.h index 2caba5c85..fc0aa055a 100644 --- a/src/node_arena.h +++ b/src/node_arena.h @@ -37,6 +37,8 @@ namespace jinja2 { +inline namespace v2 +{ // How much a handle is checked before it is followed (CMake JINJA2CPP_NODEREF_CHECKS): // 0 (OFF): never. 1 (ON, the Release default): Seal checks every handle of the tree, and // a render checks the ones that cross into another template. 2 (FULL, Debug, sanitizer and @@ -1392,6 +1394,7 @@ class NodeArena : public detail::ArenaAccess std::size_t m_symbolCount = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_NODE_ARENA_H diff --git a/src/out_stream.cpp b/src/out_stream.cpp index d64adf61d..039d07b5f 100644 --- a/src/out_stream.cpp +++ b/src/out_stream.cpp @@ -13,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ void OutStream::Flush() { if (!m_writer) @@ -262,4 +264,5 @@ void OutStream::WriteValueTo(const InternalValue& val) template void OutStream::WriteValueTo(const InternalValue& val); template void OutStream::WriteValueTo(const InternalValue& val); +} // namespace v2 } // namespace jinja2 diff --git a/src/out_stream.h b/src/out_stream.h index a080c4de8..432e41b2e 100644 --- a/src/out_stream.h +++ b/src/out_stream.h @@ -11,6 +11,8 @@ namespace jinja2 { +inline namespace v2 +{ // Where a render writes its output. A render to a string, the common case, gathers the // output in a small buffer and appends it to the string a buffer at a time: a fragment // costs a bounds check and a memcpy, with no virtual call (docs/tasks/0113). Any other @@ -171,6 +173,7 @@ TargetString RenderToString(Callback* callback, Fn&& render) return result; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_OUT_STREAM_H diff --git a/src/python_format.cpp b/src/python_format.cpp index c6739a14c..ca83ad2bd 100644 --- a/src/python_format.cpp +++ b/src/python_format.cpp @@ -21,6 +21,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -864,4 +866,5 @@ std::string CompiledPercentFormat::Format(const InternalValue* args, size_t coun return Formatter(args, count, false).Run(m_format, m_pieces); } +} // namespace v2 } // namespace jinja2 diff --git a/src/python_format.h b/src/python_format.h index 6c1e35dc8..77487aefa 100644 --- a/src/python_format.h +++ b/src/python_format.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ // Python's printf-style formatting, `format % values`, for a UTF-8 format string. `values` is // what Python's right operand is: a tuple (a list marked as tuple) of positional arguments, a @@ -78,6 +80,7 @@ class CompiledPercentFormat std::vector m_pieces; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_PYTHON_FORMAT_H diff --git a/src/recursion_guard.cpp b/src/recursion_guard.cpp index df22bb284..6716709a4 100644 --- a/src/recursion_guard.cpp +++ b/src/recursion_guard.cpp @@ -29,6 +29,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { // Stack left free for the work between two checks: a filter, a value conversion, the @@ -104,4 +106,5 @@ bool StackNearlyExhaustedSlow(std::uintptr_t frame) } } // namespace detail +} // namespace v2 } // namespace jinja2 diff --git a/src/recursion_guard.h b/src/recursion_guard.h index 4848d66cf..6f8c617b9 100644 --- a/src/recursion_guard.h +++ b/src/recursion_guard.h @@ -11,6 +11,8 @@ namespace jinja2 { +inline namespace v2 +{ // Limits that keep hostile or runaway templates from overflowing the stack (docs/tasks/0003). // @@ -189,6 +191,7 @@ class RenderDepthGuard DepthGuard m_guard; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RECURSION_GUARD_H diff --git a/src/render_context.h b/src/render_context.h index f4b56a657..9bb7216f9 100644 --- a/src/render_context.h +++ b/src/render_context.h @@ -51,6 +51,8 @@ namespace jinja2 { +inline namespace v2 +{ template class TemplateImpl; @@ -1987,6 +1989,7 @@ class HomeCall int m_debugExceptions = 0; #endif }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RENDER_CONTEXT_H diff --git a/src/render_workspace.cpp b/src/render_workspace.cpp index dc4c98897..714f1eb86 100644 --- a/src/render_workspace.cpp +++ b/src/render_workspace.cpp @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ RenderWorkspace::RenderWorkspace() { m_loopFrames.reserve(MaxLoopFrames); @@ -70,4 +72,5 @@ void RenderWorkspace::EmptyMaps(RenderMaps& maps, bool keepTables) noexcept } } } +} // namespace v2 } // namespace jinja2 diff --git a/src/render_workspace.h b/src/render_workspace.h index d6cace44b..8b9408d84 100644 --- a/src/render_workspace.h +++ b/src/render_workspace.h @@ -16,6 +16,8 @@ namespace jinja2 { +inline namespace v2 +{ struct GlobalsSnapshot; namespace detail @@ -171,6 +173,7 @@ class RenderWorkspace std::vector> m_maps; std::size_t m_lentMaps = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RENDER_WORKSPACE_H diff --git a/src/renderer.h b/src/renderer.h index 4368339d9..7ba7cfef3 100644 --- a/src/renderer.h +++ b/src/renderer.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ class IRendererBase : public ArenaNode { public: @@ -181,6 +183,7 @@ inline NodeRef MakeExpressionRenderer(NodeArena& nodes, NodeRef(expr); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RENDERER_H diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index 16f26ecdf..b948f76d1 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -8,6 +8,7 @@ #include "recursion_guard.h" #include "render_context.h" #include "string_builder.h" +#include "template_dict.h" #include "undefined.h" #include "unicode_printable.h" #include "value_visitors.h" @@ -28,10 +29,15 @@ #include #include #include +#include using namespace std::string_literals; -namespace jinja2::filters +namespace jinja2 +{ +inline namespace v2 +{ +namespace filters { struct PrettyPrinter : visitors::BaseVisitor { @@ -42,6 +48,55 @@ struct PrettyPrinter : visitors::BaseVisitor { } + // pprint's _safe_key: `a < b` when Python can order the two keys, else by the name of + // their types (str(type)), so that {1: 'a', 'b': 2} prints with the int first. Two keys + // of one type that cannot be ordered (None and None) stay where they are, as Python + // compares their ids + static bool SafeKeyLess(const InternalValue& left, const InternalValue& right) + { + const auto isNumber = [](const InternalValue& val) { + return val.Kind() == ValueKind::Int || val.Kind() == ValueKind::Double || val.Kind() == ValueKind::Bool; + }; + const auto isSequence = [](const InternalValue& val) { return val.Kind() == ValueKind::List || val.Kind() == ValueKind::KeyValuePair; }; + if ((isNumber(left) && isNumber(right)) || (IsStringValue(left) && IsStringValue(right)) || (isSequence(left) && isSequence(right))) + { + try + { + return ConvertToBool(Apply2(left, right, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive)); + } + catch (const std::runtime_error&) // NOLINT(bugprone-empty-catch): the fallback below is the handling + { + // Tuples with items Python cannot order: ordered by type like the rest + } + } + return std::strcmp(Apply(left), Apply(right)) < 0; + } + + // The entries in the order pprint prints them: by key. String keys sort by code point, + // which is UTF-8 byte order + static void SortEntriesByKey(std::vector& entries) + { + if (!std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); })) + { + std::stable_sort(entries.begin(), entries.end(), [](const KeyValuePair& left, const KeyValuePair& right) { return SafeKeyLess(left.key, right.key); }); + return; + } + std::vector> order; + order.reserve(entries.size()); + for (size_t idx = 0; idx != entries.size(); ++idx) + { + order.emplace_back(AsString(entries[idx].key), idx); + } + std::sort(order.begin(), order.end()); + std::vector sorted; + sorted.reserve(entries.size()); + for (const auto& item : order) + { + sorted.push_back(std::move(entries[item.second])); + } + entries = std::move(sorted); + } + // A value as pprint prints it: Markup as Markup('...') static std::string Print(const InternalValue& val, const RenderContext* context) { @@ -88,11 +143,11 @@ struct PrettyPrinter : visitors::BaseVisitor fmt::format_to(os, "{{"); // Python's pprint sorts dict keys; UTF-8 byte order is code point order - auto keys = map.GetKeys(); - std::sort(keys.begin(), keys.end()); + auto entries = map.GetEntries(); + SortEntriesByKey(entries); bool isFirst = true; - for (auto& k : keys) + for (auto& entry : entries) { if (isFirst) { @@ -103,8 +158,8 @@ struct PrettyPrinter : visitors::BaseVisitor fmt::format_to(os, ", "); } - fmt::format_to(os, "{}: ", Repr(k)); - fmt::format_to(os, "{}", Print(map.GetValueByName(k), m_context)); + fmt::format_to(os, "{}: ", Print(entry.key, m_context)); + fmt::format_to(os, "{}", Print(entry.value, m_context)); } fmt::format_to(os, "}}"); @@ -118,7 +173,7 @@ struct PrettyPrinter : visitors::BaseVisitor std::string str; auto os = std::back_inserter(str); - fmt::format_to(os, "({}, ", Repr(kwPair.key)); + fmt::format_to(os, "({}, ", Print(kwPair.key, m_context)); fmt::format_to(os, "{})", Print(kwPair.value, m_context)); return str; @@ -428,18 +483,7 @@ class PythonJsonWriter } else if (const auto* d = GetIf(&value)) { - if (std::isnan(*d)) - { - m_out += "NaN"; - } - else if (std::isinf(*d)) - { - m_out += *d < 0 ? "-Infinity" : "Infinity"; - } - else - { - m_out += visitors::PythonFloatText(*d).View(); - } + m_out += JsonFloatText(*d); } else if (auto str = GetAsSameString(std::string(), value)) { @@ -447,7 +491,7 @@ class PythonJsonWriter } else if (const auto* pair = GetIf(&value)) { - InternalValueList items{ InternalValue(pair->key), pair->value }; + InternalValueList items{ pair->key, pair->value }; WriteContainer('[', ']', items, level, [this, level](const InternalValue& item) { WriteValue(item, level + 1); }); } else if (auto list = AsList(value)) @@ -462,12 +506,25 @@ class PythonJsonWriter throw std::runtime_error("Object of type "s + objectClass + " is not JSON serializable"); } // sort_keys: Python orders str keys by code point, which is UTF-8 byte order - auto keys = map->GetKeys(); - std::sort(keys.begin(), keys.end()); - WriteContainer('{', '}', keys, level, [this, map, level](const std::string& key) { - WriteString(key); + auto entries = map->GetEntries(); + std::vector> order; + order.reserve(entries.size()); + if (std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); })) + { + for (size_t idx = 0; idx != entries.size(); ++idx) + { + order.emplace_back(AsString(entries[idx].key), idx); + } + std::sort(order.begin(), order.end()); + } + else + { + OrderNonStringKeys(entries, order); + } + WriteContainer('{', '}', order, level, [this, &entries, level](const std::pair& item) { + WriteString(item.first); m_out += ": "; - WriteValue(map->GetValueByName(key), level + 1); + WriteValue(entries[item.second].value, level + 1); }); } else @@ -476,6 +533,75 @@ class PythonJsonWriter } } + // How json.dumps writes a float, also as a key + static std::string JsonFloatText(double val) + { + if (std::isnan(val)) + { + return "NaN"; + } + if (std::isinf(val)) + { + return val < 0 ? "-Infinity" : "Infinity"; + } + return std::string(visitors::PythonFloatText(val).View()); + } + + // Python's `a < b` for two keys that json.dumps accepts: numbers (bool included) order + // among themselves, strings among themselves, and anything else is a TypeError + static bool KeyLess(const InternalValue& left, const InternalValue& right) + { + const bool leftString = IsStringValue(left); + const bool rightString = IsStringValue(right); + if (leftString && rightString) + { + return AsString(left) < AsString(right); + } + if (leftString || rightString || left.IsNone() || right.IsNone()) + { + throw std::runtime_error("'<' not supported between instances of '"s + Apply(left) + "' and '" + Apply(right) + "'"); + } + return ConvertToBool(Apply2(left, right, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive)); + } + + // The keys of a mapping with a key that is not a string, in sort_keys order, with the JSON + // spelling of each: 1 is "1", 2.5 is "2.5", True is "true" and None is "null". A tuple or + // an object is no JSON key; keys of types Python cannot compare (a str and an int) are a TypeError + static void OrderNonStringKeys(const std::vector& entries, std::vector>& order) + { + for (size_t idx = 0; idx != entries.size(); ++idx) + { + const auto& key = entries[idx].key; + std::string text; + if (const auto* i = GetIf(&key)) + { + text = std::to_string(*i); + } + else if (const auto* b = GetIf(&key)) + { + text = *b ? "true" : "false"; + } + else if (const auto* d = GetIf(&key)) + { + text = JsonFloatText(*d); + } + else if (key.IsNone()) + { + text = "null"; + } + else if (IsStringValue(key)) + { + text = AsString(key); + } + else + { + throw std::runtime_error("keys must be str, int, float, bool or None, not "s + Apply(key)); + } + order.emplace_back(std::move(text), idx); + } + std::stable_sort(order.begin(), order.end(), [&entries](const auto& left, const auto& right) { return KeyLess(entries[left.second].key, entries[right.second].key); }); + } + RenderContext& m_context; std::optional m_indent; StringBuilder m_out; @@ -784,7 +910,15 @@ class YamlWriter // namespace(), cycler() and joiner() are objects, not data throw std::runtime_error("Object of type "s + objectClass + " is not YAML serializable"); } - const auto keys = map.GetKeys(); + // A key that is not a string is written as its str() spelling (DictKeyText), as at the + // public boundary (docs/tasks/0036) + const auto entries = map.GetEntries(); + std::vector keys; + keys.reserve(entries.size()); + for (const auto& entry : entries) + { + keys.push_back(IsStringValue(entry.key) ? AsString(entry.key) : DictKeyText(entry.key)); + } if (keys.empty()) { StartNode(pos, indent, false); @@ -792,8 +926,9 @@ class YamlWriter return; } bool isFirst = true; - for (const auto& key : keys) + for (size_t idx = 0; idx != keys.size(); ++idx) { + const auto& key = keys[idx]; if (isFirst) { StartNode(pos, indent, true); @@ -816,7 +951,7 @@ class YamlWriter WriteString(key); } m_out.push_back(':'); - WriteNode(map.GetValueByName(key), indent + 2, Position::MappingValue); + WriteNode(entries[idx].value, indent + 2, Position::MappingValue); } } @@ -952,13 +1087,18 @@ InternalValue XmlAttrFilter::Filter(const InternalValue& baseVal, RenderContext& // Port of Jinja2's do_xmlattr: key="escaped str(value)" in the mapping's order, None and // undefined values left out (and callables, which have no str() here) std::string result; - for (auto& key : map->GetKeys()) + for (auto& [keyValue, value] : map->GetEntries()) { - auto value = map->GetValueByName(key); if (IsEmpty(value) || GetIf(&value)) { continue; } + // Jinja2 searches the key for whitespace with a regular expression, which only takes text + if (!IsStringValue(keyValue)) + { + throw std::runtime_error("expected string or bytes-like object, got '"s + Apply(keyValue) + "'"); + } + const auto key = AsString(keyValue); // Jinja2 rejects keys with whitespace, "/", ">" or "=" if (std::any_of(key.begin(), key.end(), [](char ch) { return std::strchr(" \t\n\r\f\v/>=", ch) != nullptr && ch != 0; })) { @@ -978,4 +1118,6 @@ InternalValue XmlAttrFilter::Filter(const InternalValue& baseVal, RenderContext& return resultVal; } -} // namespace jinja2::filters +} // namespace filters +} // namespace v2 +} // namespace jinja2 diff --git a/src/slot_frame.h b/src/slot_frame.h index 037bea479..b134123c4 100644 --- a/src/slot_frame.h +++ b/src/slot_frame.h @@ -25,6 +25,8 @@ namespace jinja2 { +inline namespace v2 +{ // A name's slot in the frame of its unit; Dynamic for a name looked up by name struct SlotIndex { @@ -152,6 +154,7 @@ struct ScopeView bool operator==(const ScopeView& other) const { return view == other.view && scopeIndex == other.scopeIndex && linkBelow == other.linkBelow; } bool operator!=(const ScopeView& other) const { return !(*this == other); } }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_SLOT_FRAME_H diff --git a/src/statements.cpp b/src/statements.cpp index 6c40b51c9..f7699473c 100644 --- a/src/statements.cpp +++ b/src/statements.cpp @@ -47,6 +47,8 @@ using namespace std::string_literals; namespace jinja2 { +inline namespace v2 +{ void ForStatement::Render(OutStream& os, RenderContext& values) { @@ -173,7 +175,7 @@ void AssignTo(TargetNodes target, InternalValue value, Sink& sink, RenderContext InternalValueList items; if (auto* pair = GetIf(&value)) { - items.emplace_back(TargetString(pair->key)); + items.push_back(pair->key); items.push_back(pair->value); } else @@ -432,7 +434,7 @@ void AssignLoopTarget(TargetNodes target, const InternalValue& item, ScopeRef sc { slots.items = { &scope.ForName(NameOf(target[1], values.Nodes())), &scope.ForName(NameOf(target[2], values.Nodes())) }; } - *slots.items[0] = TargetString(pair->key); + *slots.items[0] = pair->key; *slots.items[1] = pair->value; return; } @@ -1202,7 +1204,7 @@ void ForStatement::RenderLoopInSlots(const InternalValue& loopVal, OutStream& os else if (const auto* pair = isPair ? GetIf(&curValue) : nullptr) { // `for k, v in d|dictsort` (or d.items()): no list of items to unpack through - values.BindSlot(slots[target[1].slot.value], target[1].hash, TargetString(pair->key)); + values.BindSlot(slots[target[1].slot.value], target[1].hash, pair->key); values.BindSlot(slots[target[2].slot.value], target[2].hash, pair->value); } else @@ -2642,4 +2644,5 @@ void BindUndefinedStatement::Render(OutStream& /*os*/, RenderContext& values) } } } +} // namespace v2 } // namespace jinja2 diff --git a/src/statements.h b/src/statements.h index 82c7532db..32da9d31c 100644 --- a/src/statements.h +++ b/src/statements.h @@ -22,6 +22,8 @@ namespace jinja2 { +inline namespace v2 +{ class Statement : public IRendererBase { public: @@ -962,6 +964,7 @@ class BindUndefinedStatement final : public Statement ArenaSpan m_names; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_STATEMENTS_H diff --git a/src/string_builder.h b/src/string_builder.h index 2278b1437..752e2876c 100644 --- a/src/string_builder.h +++ b/src/string_builder.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ // The text of a string value being built: the first 500 characters on the stack, so that the // finished value allocates once, its object holding the characters (docs/tasks/0151). Code that @@ -64,6 +66,7 @@ class StringBuilder fmt::basic_memory_buffer m_buffer; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_STRING_BUILDER_H diff --git a/src/string_converter_filter.cpp b/src/string_converter_filter.cpp index f7deaea87..be0269f6d 100644 --- a/src/string_converter_filter.cpp +++ b/src/string_converter_filter.cpp @@ -35,7 +35,11 @@ namespace ba = boost::algorithm; using namespace std::string_literals; -namespace jinja2::filters +namespace jinja2 +{ +inline namespace v2 +{ +namespace filters { template @@ -1732,9 +1736,9 @@ InternalValue StringConverter::ApplyUrlEncode(const InternalValue& baseVal, Rend }; if (auto map = AsMap(baseVal)) { - for (auto& key : map->GetKeys()) + for (auto& [key, value] : map->GetEntries()) { - appendPair(InternalValue(key), map->GetValueByName(key)); + appendPair(key, value); } } else if (auto list = AsList(baseVal)) @@ -2217,9 +2221,15 @@ InternalValue StringConverter::Filter(const InternalValue& baseVal, RenderContex return resultVal; } -} // namespace jinja2::filters +} // namespace filters +} // namespace v2 +} // namespace jinja2 -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { namespace { @@ -2273,4 +2283,6 @@ int CompareLowerUnicode(std::wstring_view left, std::wstring_view right) { return CompareLower(left, right); } -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 diff --git a/src/template.cpp b/src/template.cpp index f4a119336..ab25b2544 100644 --- a/src/template.cpp +++ b/src/template.cpp @@ -21,6 +21,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { template @@ -189,4 +191,5 @@ template class BasicTemplate; template class BasicTemplate; #endif +} // namespace v2 } // namespace jinja2 diff --git a/src/template_dict.cpp b/src/template_dict.cpp new file mode 100644 index 000000000..e28dad6b8 --- /dev/null +++ b/src/template_dict.cpp @@ -0,0 +1,490 @@ +#include "template_dict.h" + +#include "internal_value.h" +#include "recursion_guard.h" +#include "value_visitors.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace jinja2 +{ +inline namespace v2 +{ +namespace +{ +// The kind byte after the mark. Built from char constants, never from string literals: a +// token holds NUL bytes, and "\xFFf" would read as the single character \xFFF +constexpr char KindInt = 'n'; +constexpr char KindFloat = 'g'; +constexpr char KindNone = 'N'; +constexpr char KindTuple = 't'; +constexpr char KindRange = 'r'; +constexpr char KindString = 's'; + +// The type of a value Python's hash() rejects, also inside a tuple +struct UnhashableTypeGetter : visitors::BaseVisitor +{ + using visitors::BaseVisitor::operator(); + + explicit UnhashableTypeGetter(uint64_t& items) + : m_items(items) + { + } + + const char* operator()(const ListRef& list) const + { + // A tuple can nest deeper than the stack: Python raises RecursionError + CheckStack(); + if (list.GetRangeInfo()) + { + return nullptr; + } + if (!list.IsTuple()) + { + return "list"; + } + // A lazy tuple (('x',) * 2**40) reports items it never stores: the walk is held to + // MaxSequenceSize items over all its tuples, like any sequence the template builds + m_items += list.GetSize().value_or(0); + CheckSequenceSize(m_items, "a dict key"); + const char* result = nullptr; + list.ForEach([this, &result](const InternalValue& item) { + result = Apply(item, m_items); + return result == nullptr; + }); + return result; + } + const char* operator()(const MapRef& map) const + { + return map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst ? nullptr : "dict"; + } + const char* operator()(const KeyValuePair& pair) const { return Apply(pair.value, m_items); } + +private: + uint64_t& m_items; +}; + +template +void AppendBytes(std::string& out, T value) +{ + char bytes[sizeof(T)]; + std::memcpy(bytes, &value, sizeof(T)); + out.append(bytes, sizeof(T)); +} + +void AppendInt(std::string& out, int64_t value) +{ + out.push_back(DictKeyMark); + out.push_back(KindInt); + AppendBytes(out, value); +} + +// The number of items of range(start, stop, step), for a step that is not zero +uint64_t RangeLength(const RangeInfo& range) +{ + if (range.step > 0 ? range.stop <= range.start : range.stop >= range.start) + { + return 0; + } + // Unsigned, so that the distance of two int64 values cannot overflow + if (range.step > 0) + { + const auto distance = static_cast(range.stop) - static_cast(range.start); + return ((distance - 1) / static_cast(range.step)) + 1; + } + const auto distance = static_cast(range.start) - static_cast(range.stop); + const auto stride = uint64_t{ 0 } - static_cast(range.step); + return ((distance - 1) / stride) + 1; +} + +// Appends the token of a value, and returns the type that made it unhashable instead +struct KeyTokenBuilder +{ + KeyTokenBuilder(std::string& out, bool& hasObject) + : m_out(out) + , m_hasObject(hasObject) + { + } + + template + const char* operator()(const std::basic_string& str) const + { + AddString(std::basic_string_view(str)); + return nullptr; + } + template + const char* operator()(const std::basic_string_view& str) const + { + AddString(str); + return nullptr; + } + const char* operator()(bool val) const + { + AppendInt(m_out, val ? 1 : 0); + return nullptr; + } + const char* operator()(int64_t val) const + { + AppendInt(m_out, val); + return nullptr; + } + const char* operator()(double val) const + { + // 2^63: the first double that is no int64 + constexpr double limit = 9223372036854775808.0; + if (val == std::floor(val) && val >= -limit && val < limit) + { + // 1.0 is the key 1, and -0.0 the key 0 + AppendInt(m_out, static_cast(val)); + return nullptr; + } + // Python compares NaN by identity; one NaN key here, a deliberate divergence + const auto number = std::isnan(val) ? std::numeric_limits::quiet_NaN() : val; + m_out.push_back(DictKeyMark); + m_out.push_back(KindFloat); + AppendBytes(m_out, number); + return nullptr; + } + const char* operator()(NoneValue /*val*/) const + { + m_out.push_back(DictKeyMark); + m_out.push_back(KindNone); + return nullptr; + } + const char* operator()(const ListRef& list) const + { + CheckStack(); + if (const auto* range = list.GetRangeInfo()) + { + // Equal ranges are equal keys: an empty range is one, and a single item has no step + auto length = RangeLength(*range); + auto start = range->start; + auto step = range->step; + if (length == 0) + { + start = 0; + step = 0; + } + else if (length == 1) + { + step = 1; + } + m_out.push_back(DictKeyMark); + m_out.push_back(KindRange); + AppendBytes(m_out, length); + AppendBytes(m_out, start); + AppendBytes(m_out, step); + return nullptr; + } + if (!list.IsTuple()) + { + return "list"; + } + // A lazy tuple (('x',) * 2**40) reports items it never stores; its token would, so it + // is held to MaxSequenceSize like any sequence the template builds + const auto size = list.GetSize().value_or(0); + CheckSequenceSize(size, "a dict key"); + m_out.push_back(DictKeyMark); + m_out.push_back(KindTuple); + AppendBytes(m_out, static_cast(size)); + const char* result = nullptr; + list.ForEach([this, &result](const InternalValue& item) { + CheckSequenceSize(m_out.size(), "a dict key"); + result = AddElement(item); + return result == nullptr; + }); + return result; + } + const char* operator()(const MapRef& map) const + { + if (map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst) + { + AddObject(); + return nullptr; + } + return "dict"; + } + // A dictsort pair is the tuple (key, value) + const char* operator()(const KeyValuePair& pair) const + { + CheckStack(); + m_out.push_back(DictKeyMark); + m_out.push_back(KindTuple); + AppendBytes(m_out, uint32_t{ 2 }); + if (const char* type = AddElement(pair.key)) + { + return type; + } + return AddElement(pair.value); + } + // Undefined values, callables and renderers + template + const char* operator()(const T& /*val*/) const + { + AddObject(); + return nullptr; + } + +private: + // A tuple item: the length of its token, then the token + [[nodiscard]] const char* AddElement(const InternalValue& item) const + { + const auto lengthAt = m_out.size(); + AppendBytes(m_out, uint32_t{ 0 }); + const auto start = m_out.size(); + if (const char* type = Apply(item, m_out, m_hasObject)) + { + return type; + } + const auto length = static_cast(m_out.size() - start); + std::memcpy(&m_out[lengthAt], &length, sizeof(length)); + return nullptr; + } + + // Objects, undefined values and callables have no key of their own yet (docs/tasks/0212): + // they add nothing, so as a key each stands for the empty string, as dict keys did before, + // and a lookup by one finds nothing (MakeDictKey's hasObject) + void AddObject() const { m_hasObject = true; } + + template + void AddString(std::basic_string_view str) const + { + if constexpr (std::is_same_v) + { + if (!str.empty() && str.front() == DictKeyMark) + { + m_out.push_back(DictKeyMark); + m_out.push_back(KindString); + } + m_out.append(str); + } + else + { + // Keys are narrow UTF-8 whatever the template's character type + const auto text = ConvertString(str); + AddString(std::string_view(text)); + } + } + + std::string& m_out; + bool& m_hasObject; +}; + +std::string EscapeString(const std::string& name) +{ + std::string token; + token.reserve(name.size() + 2); + token.push_back(DictKeyMark); + token.push_back(KindString); + token += name; + return token; +} + +bool NeedsEscape(const std::string& name) +{ + return !name.empty() && name.front() == DictKeyMark; +} + +// Whether a stored name is a token that has a spelling of its own, not an escaped string +bool IsSpelledToken(const std::string& stored) +{ + return stored.size() >= 2 && stored.front() == DictKeyMark && stored[1] != KindString; +} +} // namespace + +const char* UnhashableType(const InternalValue& value) +{ + uint64_t items = 0; + return Apply(value, items); +} + +std::optional MakeDictKey(const InternalValue& key, const char** unhashableType, bool* hasObject) +{ + std::string token; + bool objectFound = false; + const char* type = Apply(key, token, objectFound); + if (hasObject) + { + *hasObject = objectFound; + } + if (type) + { + if (unhashableType) + { + *unhashableType = type; + } + return std::nullopt; + } + return token; +} + +void ThrowUnhashable(const char* type) +{ + throw std::runtime_error(std::string("unhashable type: '") + (type ? type : "object") + "'"); +} + +std::string DictKeyText(const InternalValue& key) +{ + if (IsStringValue(key)) + { + return AsString(key); + } + std::string text; + Apply>(key, text, false); + return text; +} + +TemplateDict::TemplateDict(InternalDict&& items) + : m_items(std::move(items)) +{ +#ifndef NDEBUG + for (const auto& item : m_items) + { + assert(!NeedsEscape(item.first) && "a name never starts with the mark of a dict key token"); + } +#endif +} + +TemplateDict::TemplateDict(const TemplateDict& other) + : m_items(other.m_items) + , m_keys(other.m_keys ? std::make_unique(*other.m_keys) : nullptr) +{ +} + +TemplateDict& TemplateDict::operator=(const TemplateDict& other) +{ + if (this != &other) + { + TemplateDict copy(other); + *this = std::move(copy); + } + return *this; +} + +TemplateDict::const_iterator TemplateDict::FindString(const std::string& name) const +{ + return NeedsEscape(name) ? m_items.find(EscapeString(name)) : m_items.find(name); +} + +TemplateDict::iterator TemplateDict::FindString(const std::string& name) +{ + return NeedsEscape(name) ? m_items.find(EscapeString(name)) : m_items.find(name); +} + +TemplateDict::const_iterator TemplateDict::Find(const InternalValue& key) const +{ + bool hasObject = false; + auto token = MakeDictKey(key, nullptr, &hasObject); + return token && !hasObject ? m_items.find(*token) : m_items.end(); +} + +TemplateDict::iterator TemplateDict::Find(const InternalValue& key) +{ + bool hasObject = false; + auto token = MakeDictKey(key, nullptr, &hasObject); + return token && !hasObject ? m_items.find(*token) : m_items.end(); +} + +TemplateDict::iterator TemplateDict::Set(const InternalValue& key, InternalValue value) +{ + const char* unhashable = nullptr; + auto token = MakeDictKey(key, &unhashable); + if (!token) + { + ThrowUnhashable(unhashable); + } + auto [pos, inserted] = m_items.insert_or_assign(std::move(*token), std::move(value)); + if (inserted && IsSpelledToken(pos->first)) + { + try + { + if (!m_keys) + { + m_keys = std::make_unique(); + } + m_keys->emplace(pos->first, key); + } + catch (...) + { + m_items.erase(pos); + throw; + } + } + return pos; +} + +TemplateDict::iterator TemplateDict::SetString(std::string name, InternalValue value) +{ + if (NeedsEscape(name)) + { + return m_items.insert_or_assign(EscapeString(name), std::move(value)).first; + } + return m_items.insert_or_assign(std::move(name), std::move(value)).first; +} + +std::pair TemplateDict::TryEmplace(const InternalValue& key, InternalValue value) +{ + const char* unhashable = nullptr; + auto token = MakeDictKey(key, &unhashable); + if (!token) + { + ThrowUnhashable(unhashable); + } + if (auto pos = m_items.find(*token); pos != m_items.end()) + { + return { pos, false }; + } + return { Set(key, std::move(value)), true }; +} + +TemplateDict::iterator TemplateDict::Erase(const_iterator pos) +{ + if (m_keys && IsSpelledToken(pos->first)) + { + m_keys->erase(pos->first); + } + return m_items.erase(pos); +} + +void TemplateDict::Clear() +{ + m_items.clear(); + m_keys.reset(); +} + +InternalValue TemplateDict::KeyOf(const std::string& stored) const +{ + if (!NeedsEscape(stored)) + { + return InternalValue(stored); + } + if (m_keys) + { + if (auto spelling = m_keys->find(stored); spelling != m_keys->end()) + { + return spelling->second; + } + } + // An escaped string; a token without a spelling is read as the string it is + if (stored.size() >= 2 && stored[1] == KindString) + { + return InternalValue(stored.substr(2)); + } + return InternalValue(stored); +} + +} // namespace v2 +} // namespace jinja2 diff --git a/src/template_dict.h b/src/template_dict.h new file mode 100644 index 000000000..58f5297b7 --- /dev/null +++ b/src/template_dict.h @@ -0,0 +1,117 @@ +#ifndef JINJA2CPP_SRC_TEMPLATE_DICT_H +#define JINJA2CPP_SRC_TEMPLATE_DICT_H + +#include "internal_value.h" + +#include +#include +#include +#include +#include +#include + +namespace jinja2 +{ +inline namespace v2 +{ +// The dict a template owns (dict literals, dict(), namespace(), the result of the dict methods): +// Python keys, which are any hashable value, over the string keys of InternalDict (docs/tasks/0036). +// +// A string key is stored as itself. Any other hashable key is stored under a canonical token +// (MakeDictKey): one binary string that is equal for keys Python finds equal (1, 1.0 and True; +// (1, 2.0) and (1.0, 2)), so the dict needs no custom hash or comparison and a lookup is one +// find. A token cannot be turned back into a key (a float would have to be parsed), so the +// first spelling of each key is kept in a side table, allocated only when the dict has a +// key that is not a string. Iteration reads the keys from it. +// +// Every key goes in through Set or SetString; Items() is const so that no code can store a raw +// string under a token's name. + +// The first byte of every token, a byte that is not valid in UTF-8. A string that starts with +// it (host data that is not valid UTF-8) is escaped, so a token is never a plain string +inline constexpr char DictKeyMark = static_cast(0xFF); + +// The type Python's hash() rejects in a value, if any: a list or a dict, also inside a tuple. +// A range hashes, and so does an object stored as a map (namespace(), cycler()) or a host object +// (a reflected struct, which Python would hold as an object, or a JSON object) +const char* UnhashableType(const InternalValue& value); + +// The token of a key, which is equal for the keys Python treats as one. Nullopt when the key +// cannot be hashed, with the type to report in `unhashableType`: Python's TypeError +// `unhashable type: 'list'`. Objects (namespace(), cycler(), host values), undefined values and +// callables have no token of their own yet (docs/tasks/0212): they read as the empty string, and +// `hasObject` tells a lookup that the token stands for no key of its own +std::optional MakeDictKey(const InternalValue& key, const char** unhashableType = nullptr, bool* hasObject = nullptr); + +// Throws Python's TypeError for a key MakeDictKey rejects +[[noreturn]] void ThrowUnhashable(const char* type); + +// str(key): how a mapping that holds only string keys (ValuesMap) spells a key it cannot keep +std::string DictKeyText(const InternalValue& key); + +class TemplateDict +{ +public: + using iterator = InternalDict::iterator; + using const_iterator = InternalDict::const_iterator; + + TemplateDict() = default; + // A dict of string keys, such as the names of keyword arguments, which never start with the mark + explicit TemplateDict(InternalDict&& items); + TemplateDict(const TemplateDict& other); + TemplateDict(TemplateDict&& other) noexcept = default; + TemplateDict& operator=(const TemplateDict& other); + TemplateDict& operator=(TemplateDict&& other) noexcept = default; + ~TemplateDict() = default; + + // The entries under their tokens: for the count and the values. Read the keys through KeyOf + [[nodiscard]] const InternalDict& Items() const { return m_items; } + [[nodiscard]] size_t size() const { return m_items.size(); } + [[nodiscard]] bool empty() const { return m_items.empty(); } + [[nodiscard]] iterator begin() { return m_items.begin(); } + [[nodiscard]] iterator end() { return m_items.end(); } + [[nodiscard]] const_iterator begin() const { return m_items.begin(); } + [[nodiscard]] const_iterator end() const { return m_items.end(); } + // Whether some key is not a string, which the public (string keyed) view has to spell + [[nodiscard]] bool HasKeyValues() const { return m_keys && !m_keys->empty(); } + + // A string key; end() when absent + [[nodiscard]] const_iterator FindString(const std::string& name) const; + [[nodiscard]] iterator FindString(const std::string& name); + // Any key; end() when absent or unhashable (the caller checks UnhashableType where Python + // raises), and for a key that holds an object, which has no token of its own yet (0212) + [[nodiscard]] const_iterator Find(const InternalValue& key) const; + [[nodiscard]] iterator Find(const InternalValue& key); + + // Python's d[key] = value: a new key is appended with this spelling, an existing key (the + // same token) keeps its position and its first spelling and takes the value. Throws + // TypeError for an unhashable key + iterator Set(const InternalValue& key, InternalValue value); + iterator SetString(std::string name, InternalValue value); + // Appends the key if it is new and keeps the value of one that exists (setdefault) + std::pair TryEmplace(const InternalValue& key, InternalValue value); + + iterator Erase(const_iterator pos); + void Clear(); + + // The key of a stored entry as a template value: the string, or the first spelling of a + // non-string key. An entry that starts with the mark and has no spelling reads as a plain string + [[nodiscard]] InternalValue KeyOf(const std::string& stored) const; + + // Dicts are equal when they hold the same entries, in any order. Tokens are canonical, so + // comparing the entries compares the keys + friend bool operator==(const TemplateDict& lhs, const TemplateDict& rhs) { return lhs.m_items == rhs.m_items; } + friend bool operator!=(const TemplateDict& lhs, const TemplateDict& rhs) { return !(lhs == rhs); } + +private: + using KeyTable = std::unordered_map; + + InternalDict m_items; + // Token to first spelling, for the entries that are not stored under their own string + std::unique_ptr m_keys; +}; + +} // namespace v2 +} // namespace jinja2 + +#endif // JINJA2CPP_SRC_TEMPLATE_DICT_H diff --git a/src/template_env.cpp b/src/template_env.cpp index 312c9f7d5..a02a5ba1f 100644 --- a/src/template_env.cpp +++ b/src/template_env.cpp @@ -26,6 +26,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -487,4 +489,5 @@ void TemplateEnv::ApplyGlobals(const std::function& fn) fn(*m_impl->globalValues); } +} // namespace v2 } // namespace jinja2 diff --git a/src/template_env_impl.h b/src/template_env_impl.h index e39adf31c..176759c19 100644 --- a/src/template_env_impl.h +++ b/src/template_env_impl.h @@ -19,7 +19,11 @@ #include #include -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // The state of a TemplateEnv. The TemplateEnv the user creates owns it; every template made in the environment holds @@ -100,6 +104,8 @@ struct TemplateEnvAccess static TemplateEnv MakeHandle(const TemplateEnv* env) { return TemplateEnv(env ? env->m_impl : nullptr); } }; -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_ENV_IMPL_H diff --git a/src/template_impl.h b/src/template_impl.h index 63c92840f..c7ba232da 100644 --- a/src/template_impl.h +++ b/src/template_impl.h @@ -63,6 +63,8 @@ namespace jinja2 { +inline namespace v2 +{ // The default globals of every render, looked up after the environment's (global_functions.cpp) extern const InternalValueMap& GetBuiltinGlobals(bool withI18n); @@ -870,6 +872,7 @@ class TemplateImpl : public ITemplateImpl MetadataInfo m_metadataInfo; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_IMPL_H diff --git a/src/template_parser.cpp b/src/template_parser.cpp index 69d6f15a2..097445c42 100644 --- a/src/template_parser.cpp +++ b/src/template_parser.cpp @@ -34,6 +34,8 @@ namespace jinja2 { +inline namespace v2 +{ NodeRef StatementsParser::TakeFrameBody(StatementInfo& info) { @@ -1952,4 +1954,5 @@ InternalValue EscapedLiteralValue(std::basic_string_view text, const std: template InternalValue EscapedLiteralValue(std::string_view text, const std::string& newlineSequence, bool& malformed); template InternalValue EscapedLiteralValue(std::wstring_view text, const std::string& newlineSequence, bool& malformed); +} // namespace v2 } // namespace jinja2 diff --git a/src/template_parser.h b/src/template_parser.h index 787960f41..fe0cc7625 100644 --- a/src/template_parser.h +++ b/src/template_parser.h @@ -50,6 +50,8 @@ namespace jinja2 { +inline namespace v2 +{ template struct ParserTraits; @@ -2391,6 +2393,7 @@ std::unordered_map ParserTraitsBase::s_tokens = { { Token::ExprEnd, UNIVERSAL_STR("}}") }, }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_PARSER_H diff --git a/src/template_slots.h b/src/template_slots.h index 21f1781f9..3d755f6b4 100644 --- a/src/template_slots.h +++ b/src/template_slots.h @@ -15,6 +15,8 @@ namespace jinja2 { +inline namespace v2 +{ // A template one render runs, by its place in the render's table of templates (0118 P4b). // The render keeps every template it loads until it ends, so a handle needs no reference @@ -165,6 +167,7 @@ class TemplateSlots std::uint8_t m_generation = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_SLOTS_H diff --git a/src/tester_factories.cpp b/src/tester_factories.cpp index 2a3c452ba..8136a2b80 100644 --- a/src/tester_factories.cpp +++ b/src/tester_factories.cpp @@ -22,6 +22,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -149,4 +151,5 @@ TesterPtr CreateTester(std::string testerName, const CallParamsInfo& params, Ren return std::make_shared(std::move(testerName), params, std::move(callable)); } +} // namespace v2 } // namespace jinja2 diff --git a/src/testers.cpp b/src/testers.cpp index caba25367..894f02c8a 100644 --- a/src/testers.cpp +++ b/src/testers.cpp @@ -5,6 +5,7 @@ #include "function_base.h" #include "internal_value.h" #include "render_context.h" +#include "template_dict.h" #include "undefined.h" #include "value_visitors.h" @@ -23,7 +24,11 @@ using namespace std::string_literals; -namespace jinja2::testers +namespace jinja2 +{ +inline namespace v2 +{ +namespace testers { Comparator::Comparator(const TesterParams& params, BinaryExpression::Operation op) @@ -406,9 +411,21 @@ bool IsValueIn(const InternalValue& baseVal, const InternalValue& seq) } else if (seqKind == TestKind::Map) { - // `key in dict` tests the keys; dict keys are always strings here + // `key in dict` tests the keys: any hashable value for a dict the template owns, strings + // for every other map auto map = AsMap(seq); - result = map && Apply(baseVal) == TestKind::String && map->HasValue(AsString(baseVal)); + if (map && Apply(baseVal) == TestKind::String) + { + result = map->HasValue(AsString(baseVal)); + } + else if (map) + { + if (const char* type = UnhashableType(baseVal)) + { + ThrowUnhashable(type); + } + result = map->FindItem(baseVal).has_value(); + } } else if (seqKind == TestKind::String) { @@ -598,4 +615,6 @@ bool UserDefinedTester::Test(const InternalValue& baseVal, RenderContext& contex return ConvertToBool(callable->GetExpressionCallable()(callParams, context)); } -} // namespace jinja2::testers +} // namespace testers +} // namespace v2 +} // namespace jinja2 diff --git a/src/testers.h b/src/testers.h index 37f906679..0c2e8298c 100644 --- a/src/testers.h +++ b/src/testers.h @@ -13,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ using TesterPtr = std::shared_ptr; using TesterParams = CallParamsInfo; @@ -117,6 +119,7 @@ class UserDefinedTester : public TesterBase InternalValue m_callable; }; } // namespace testers +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TESTERS_H diff --git a/src/undefined.cpp b/src/undefined.cpp index a397e1107..cff2aab33 100644 --- a/src/undefined.cpp +++ b/src/undefined.cpp @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -66,7 +68,7 @@ InternalValue MakeUndefined(const RenderContext* context, const InternalValue& o // A key a host map has but reads as undefined (a JSON null or an empty reflected field, // task 0047) is not missing: it stays a plain undefined, which fails no use auto map = AsMap(obj); - if (map && IsStringValue(key) && map->HasValue(AsString(key))) + if (map && (IsStringValue(key) ? map->HasValue(AsString(key)) : map->FindItem(key).has_value())) { return InternalValue(); } @@ -132,4 +134,5 @@ const char* UndefinedClassName(const UndefinedInfo& info) return "Undefined"; } +} // namespace v2 } // namespace jinja2 diff --git a/src/undefined.h b/src/undefined.h index a1ae94598..a7b1c614e 100644 --- a/src/undefined.h +++ b/src/undefined.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ // What a named lookup missed, and the policy that decides how the undefined result may be // used. It travels with the value, as the class of a Python Undefined does: a variable set @@ -102,6 +104,7 @@ inline void CheckStrictUndefined(const UndefinedValue& val) } } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_UNDEFINED_H diff --git a/src/unicode_case.h b/src/unicode_case.h index 75a163e37..0e6d46c9d 100644 --- a/src/unicode_case.h +++ b/src/unicode_case.h @@ -10,7 +10,11 @@ #include #include -namespace jinja2::unicode +namespace jinja2 +{ +inline namespace v2 +{ +namespace unicode { // cp + delta for every stride-th code point from first to last @@ -1413,6 +1417,8 @@ inline size_t ToTitle(uint32_t cp, uint32_t (&out)[3]) { return MapCase(TitleRun inline bool IsCased(uint32_t cp) { return InRanges(CasedRanges, cp); } inline bool IsCaseIgnorable(uint32_t cp) { return InRanges(CaseIgnorableRanges, cp); } -} // namespace jinja2::unicode +} // namespace unicode +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_UNICODE_CASE_H diff --git a/src/unicode_printable.h b/src/unicode_printable.h index 8162d7607..8fa21ec43 100644 --- a/src/unicode_printable.h +++ b/src/unicode_printable.h @@ -7,7 +7,11 @@ #include #include -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // True when Python's str.isprintable() holds for the code point, so repr() keeps it as is @@ -216,6 +220,8 @@ inline bool IsPythonPrintable(uint32_t cp) return cp > it->last; } -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_UNICODE_PRINTABLE_H diff --git a/src/unicode_tables.h b/src/unicode_tables.h index b43614642..fce4ba143 100644 --- a/src/unicode_tables.h +++ b/src/unicode_tables.h @@ -9,6 +9,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace unicode { @@ -877,6 +879,7 @@ inline bool IsSpace(uint32_t cp) } } // namespace unicode +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_UNICODE_TABLES_H diff --git a/src/value.cpp b/src/value.cpp index d62bdefa3..8c201df86 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -15,6 +15,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { // Every printing form of Value meets here, so none can print differently from {{ value }}: @@ -123,4 +125,5 @@ std::to_chars_result Value::ToChars(char* first, char* last) const noexcept } } +} // namespace v2 } // namespace jinja2 diff --git a/src/value_methods.cpp b/src/value_methods.cpp index f098d3879..179e7ec1b 100644 --- a/src/value_methods.cpp +++ b/src/value_methods.cpp @@ -5,6 +5,7 @@ #include "internal_value.h" #include "markup.h" #include "render_context.h" +#include "template_dict.h" #include "unicode_tables.h" #include "value_visitors.h" @@ -17,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +33,11 @@ using namespace std::string_literals; -namespace jinja2::methods +namespace jinja2 +{ +inline namespace v2 +{ +namespace methods { namespace { @@ -51,16 +57,13 @@ bool Equals(const InternalValue& lhs, const InternalValue& rhs) return ConvertToBool(Apply2(lhs, rhs, BinaryExpression::LogicalEq)); } -// Mapping keys are strings in the value model (task 0036): other keys by their printed form -std::string KeyString(const InternalValue& key) +// Python hashes a dict key before it looks at the dict: a list or a dict is a TypeError +void CheckHashable(const InternalValue& key) { - if (IsStringValue(key)) + if (const char* type = UnhashableType(key)) { - return AsString(key); + ThrowUnhashable(type); } - std::string result; - Apply>(key, result); - return result; } // Positional argument `idx`, or the keyword argument `kwName` when the method takes one @@ -1696,6 +1699,8 @@ bool ReachesThroughOpaque(const InternalValue& root, const void* target) VisitedStorage visited; // Items stay where they are while the walk runs: nothing changes a container under it std::vector pending{ { &root, false } }; + // Keys are read by value, so the walk keeps them where it can point at them + std::deque keys; // Only values that can hold a container are queued: a wide flat list costs no work list const auto queueValue = [&pending](const InternalValue& item, bool viaOpaque) { const auto kind = item.Kind(); @@ -1716,6 +1721,19 @@ bool ReachesThroughOpaque(const InternalValue& root, const void* target) } else if (auto map = AsMap(*val)) { + auto* items = map->GetMutableItems(); + // A key that is not a string keeps its first spelling, which CycleCollector follows too + if (items) + { + if (items->HasKeyValues() && visited[viaOpaque ? 1 : 0].count(items) == 0) + { + for (const auto& entry : *items) + { + keys.push_back(items->KeyOf(entry.first)); + queueValue(keys.back(), viaOpaque); + } + } + } found = QueueItems(map->GetMutableItems(), target, viaOpaque, visited, queuePairValue); } else if (const auto* callable = GetIf(val)) @@ -1893,7 +1911,7 @@ MapRef MapOf(const InternalValue& self) return *map; } -InternalDict& MutableDict(const InternalValue& self) +TemplateDict& MutableDict(const InternalValue& self) { auto* items = MapOf(self).GetMutableItems(); if (!items) @@ -1903,31 +1921,10 @@ InternalDict& MutableDict(const InternalValue& self) return *items; } -// The keys in iteration order: insertion order for the dicts the template owns -std::vector KeysOf(const MapRef& map) -{ - if (auto* items = map.GetMutableItems()) - { - std::vector keys; - keys.reserve(items->size()); - for (auto& [key, value] : *items) - { - keys.push_back(key); - } - return keys; - } - return map.GetKeys(); -} - InternalValue DictKeys(const InternalValue& self, const CallParams& params, RenderContext&) { CheckArgs(params, "keys", 0, 0); - InternalValueList result; - for (auto& key : KeysOf(MapOf(self))) - { - result.emplace_back(key); - } - return ListAdapter::CreateAdapter(std::move(result)); + return ListAdapter::CreateAdapter(MapOf(self).GetKeyValues()); } InternalValue DictValues(const InternalValue& self, const CallParams& params, RenderContext&) @@ -1935,9 +1932,9 @@ InternalValue DictValues(const InternalValue& self, const CallParams& params, Re CheckArgs(params, "values", 0, 0); const auto& map = MapOf(self); InternalValueList result; - for (auto& key : KeysOf(map)) + for (auto& entry : map.GetEntries()) { - result.push_back(map.GetValueByName(key)); + result.push_back(std::move(entry.value)); } InternalValue list = ListAdapter::CreateAdapter(std::move(result)); if (self.ShouldExtendLifetime()) @@ -1952,9 +1949,9 @@ InternalValue DictItems(const InternalValue& self, const CallParams& params, Ren CheckArgs(params, "items", 0, 0); const auto& map = MapOf(self); InternalValueList result; - for (auto& key : KeysOf(map)) + for (auto& entry : map.GetEntries()) { - result.push_back(MakeTuple({ InternalValue(key), map.GetValueByName(key) })); + result.push_back(MakeTuple({ std::move(entry.key), std::move(entry.value) })); } InternalValue list = ListAdapter::CreateAdapter(std::move(result)); if (self.ShouldExtendLifetime()) @@ -1968,10 +1965,23 @@ InternalValue DictGet(const InternalValue& self, const CallParams& params, Rende { CheckArgs(params, "get", 1, 2); const auto& map = MapOf(self); - auto key = KeyString(params.posParams[0]); - if (map.HasValue(key)) + const auto& key = params.posParams[0]; + if (IsStringValue(key)) + { + const auto name = AsString(key); + if (map.HasValue(name)) + { + return map.GetValueByName(name); + } + } + else { - return map.GetValueByName(key); + // Python hashes the key first, also when the dict has no such key + CheckHashable(key); + if (auto found = map.FindItem(key)) + { + return std::move(*found); + } } return params.posParams.size() > 1 ? params.posParams[1] : InternalValue(NoneValue()); } @@ -1980,19 +1990,22 @@ InternalValue DictSetdefault(const InternalValue& self, const CallParams& params { CheckArgs(params, "setdefault", 1, 2); auto& items = MutableDict(self); - auto key = KeyString(params.posParams[0]); - auto p = items.find(key); + const auto& key = params.posParams[0]; + CheckHashable(key); + auto p = items.Find(key); if (p != items.end()) { return p->second; } InternalValue value = params.posParams.size() > 1 ? params.posParams[1] : InternalValue(NoneValue()); + // A key that is not a string is kept too, and a tuple key can hold a container + NoteContainerStore(self, &items, key); NoteContainerStore(self, &items, value); - items[key] = value; + items.Set(key, value); return value; } -using DictUpdates = std::vector>; +using DictUpdates = std::vector>; // The key-value pairs of an iterable of pairs, d.update([(k, v), ...]) void CollectPairUpdates(const InternalValue& other, DictUpdates& updates) @@ -2012,7 +2025,9 @@ void CollectPairUpdates(const InternalValue& other, DictUpdates& updates) { Raise("dictionary update sequence element has wrong length; 2 is required"); } - updates.emplace_back(KeyString(pair[0]), pair[1]); + // A failing update leaves the dict as it was: the keys are checked before any is stored + CheckHashable(pair[0]); + updates.emplace_back(pair[0], pair[1]); } } @@ -2022,9 +2037,9 @@ void CollectPositionalUpdates(const InternalValue& other, DictUpdates& updates) // An object such as namespace() is no mapping, and not iterable either if (auto otherMap = AsMap(other); otherMap && !otherMap->GetObjectClass()) { - for (auto& key : KeysOf(*otherMap)) + for (auto& entry : otherMap->GetEntries()) { - updates.emplace_back(key, otherMap->GetValueByName(key)); + updates.emplace_back(std::move(entry.key), std::move(entry.value)); } } else @@ -2047,16 +2062,17 @@ InternalValue DictUpdate(const InternalValue& self, const CallParams& params, Re } for (const auto& [name, value] : params.kwParams) { - updates.emplace_back(name, value); + updates.emplace_back(InternalValue(name), value); } auto& items = MutableDict(self); - for (auto& [name, value] : updates) + for (auto& [key, value] : updates) { + NoteContainerStore(self, &items, key); NoteContainerStore(self, &items, value); } - for (auto& [name, value] : updates) + for (auto& [key, value] : updates) { - items[name] = std::move(value); + items.Set(key, std::move(value)); } return InternalValue(NoneValue()); } @@ -2065,18 +2081,21 @@ InternalValue DictPop(const InternalValue& self, const CallParams& params, Rende { CheckArgs(params, "pop", 1, 2); auto& items = MutableDict(self); - auto key = KeyString(params.posParams[0]); - auto p = items.find(key); + const auto& key = params.posParams[0]; + CheckHashable(key); + auto p = items.Find(key); if (p == items.end()) { if (params.posParams.size() > 1) { return params.posParams[1]; } - Raise("KeyError: '" + key + "'"); + std::string spelling; + Apply>(key, spelling, true); + Raise("KeyError: " + spelling); } auto result = p->second; - items.erase(p); + items.Erase(p); return result; } @@ -2089,8 +2108,8 @@ InternalValue DictPopitem(const InternalValue& self, const CallParams& params, R Raise("popitem(): dictionary is empty"); } auto last = std::prev(items.end()); - auto result = MakeTuple({ InternalValue(last->first), last->second }); - items.erase(last); + auto result = MakeTuple({ items.KeyOf(last->first), last->second }); + items.Erase(last); return result; } @@ -2098,10 +2117,14 @@ InternalValue DictCopy(const InternalValue& self, const CallParams& params, Rend { CheckArgs(params, "copy", 0, 0); const auto& map = MapOf(self); - InternalDict result; - for (auto& key : KeysOf(map)) + if (const auto* items = map.GetMutableItems()) + { + return CreateMapAdapter(TemplateDict(*items)); + } + TemplateDict result; + for (auto& entry : map.GetEntries()) { - result[key] = map.GetValueByName(key); + result.Set(entry.key, std::move(entry.value)); } return CreateMapAdapter(std::move(result)); } @@ -2109,7 +2132,7 @@ InternalValue DictCopy(const InternalValue& self, const CallParams& params, Rend InternalValue DictClear(const InternalValue& self, const CallParams& params, RenderContext&) { CheckArgs(params, "clear", 0, 0); - MutableDict(self).clear(); + MutableDict(self).Clear(); return InternalValue(NoneValue()); } @@ -2319,7 +2342,7 @@ void StoreItem(const InternalValue& container, const InternalValue& key, Interna auto* items = map->GetMutableItems(); if (items) { - (*items)[KeyString(key)] = std::move(value); + items->Set(key, std::move(value)); } } } @@ -2368,15 +2391,14 @@ InternalValue MakeMutable(const InternalValue& value) } if (auto map = AsMap(value)) { - InternalDict items; - for (auto& key : map->GetKeys()) + TemplateDict items; + for (auto& [key, item] : map->GetEntries()) { - auto item = map->GetValueByName(key); if (extendLifetime) { item.SetParentData(value); } - items[key] = std::move(item); + items.Set(key, std::move(item)); } return CreateMapAdapter(std::move(items)); } @@ -2388,4 +2410,6 @@ void ThrowNoAttribute(const InternalValue& obj, const std::string& name) Raise("'" + TypeName(obj) + " object' has no attribute '" + name + "'"); } -} // namespace jinja2::methods +} // namespace methods +} // namespace v2 +} // namespace jinja2 diff --git a/src/value_methods.h b/src/value_methods.h index 9d6f2df47..ff9283195 100644 --- a/src/value_methods.h +++ b/src/value_methods.h @@ -8,6 +8,8 @@ namespace jinja2 { +inline namespace v2 +{ struct CallParams; // Python's builtin methods on template values: s.upper(), l.append(x), d.items(), ... @@ -65,6 +67,7 @@ bool IsContainer(const InternalValue& value); [[noreturn]] void ThrowNoAttribute(const InternalValue& obj, const std::string& name); } // namespace methods +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_VALUE_METHODS_H diff --git a/src/value_visitors.h b/src/value_visitors.h index 312a203dc..8d6da9490 100644 --- a/src/value_visitors.h +++ b/src/value_visitors.h @@ -38,6 +38,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -847,21 +849,20 @@ void ValueRendererBase::operator()(const MapRef& map) const // Python prints dicts in their own order: ValuesMap and InternalDict keep insertion order, reflected // structs report sorted keys - auto keys = map.GetKeys(); + auto entries = map.GetEntries(); - ValueRenderer keyRenderer(*m_os, true); AppendAscii("{"); bool isFirst = true; - for (auto& key : keys) + for (auto& [key, value] : entries) { if (!isFirst) { AppendAscii(", "); } isFirst = false; - keyRenderer(key); + RenderRepr(key, &containers); AppendAscii(": "); - RenderRepr(map.GetValueByName(key), &containers); + RenderRepr(value, &containers); } AppendAscii("}"); if (objectClass) @@ -876,7 +877,7 @@ void ValueRendererBase::operator()(const KeyValuePair& pair) const { // dict items are (key, value) tuples in Python AppendAscii("("); - ValueRenderer(*m_os, true)(pair.key); + RenderRepr(pair.key, m_containers); AppendAscii(", "); RenderRepr(pair.value, m_containers); AppendAscii(")"); @@ -1755,9 +1756,10 @@ struct BinaryMathOperation : BaseVisitor<> { CheckStack(); equal = true; - for (auto& key : left.GetKeys()) + for (auto& [key, value] : left.GetEntries()) { - if (!right.HasValue(key) || !ConvertToBool(Apply2(left.GetValueByName(key), right.GetValueByName(key), BinaryExpression::LogicalEq))) + const auto other = right.FindItem(key); + if (!other || !ConvertToBool(Apply2(value, *other, BinaryExpression::LogicalEq))) { equal = false; break; @@ -2082,6 +2084,7 @@ JINJA2CPP_HOT_INLINE InternalValue ApplyToNumbers(const InternalValue& left, con } // namespace visitors +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_VALUE_VISITORS_H diff --git a/test/abi_symbols.cmake b/test/abi_symbols.cmake new file mode 100644 index 000000000..71bdc935c --- /dev/null +++ b/test/abi_symbols.cmake @@ -0,0 +1,48 @@ +# Fails when the library defines a jinja2 symbol outside the inline ABI namespace (docs/tasks/0076). +# cmake -DNM= -DLIBRARY= -P test/abi_symbols.cmake +# Every mangled jinja2:: name must start jinja2::v2:: (`6jinja22v2`), and the vendored +# polymorphic/indirect types must not be left in a global `xyz`, where a user's own copy of +# the upstream headers would collide with them. scripts/abi_namespace.py keeps the sources +# in that shape; this checks what the compiler actually emitted. +foreach (_var NM LIBRARY) + if (NOT ${_var}) + message(FATAL_ERROR "${_var} is required") + endif () +endforeach () +# Defined symbols only: a static library repeats every reference in each object +execute_process(COMMAND "${NM}" --defined-only "${LIBRARY}" OUTPUT_VARIABLE _symbols ERROR_VARIABLE _err RESULT_VARIABLE _rc) +if (NOT _rc EQUAL 0) + execute_process(COMMAND "${NM}" "${LIBRARY}" OUTPUT_VARIABLE _symbols ERROR_VARIABLE _err RESULT_VARIABLE _rc) +endif () +if (NOT _rc EQUAL 0) + message(FATAL_ERROR "${NM} ${LIBRARY} failed (${_rc}): ${_err}") +endif () +# Mach-O prefixes every symbol with an extra underscore; the match starts after it +string(REGEX MATCHALL "_Z[^ \n]*" _syms "${_symbols}") +set(_tagged ${_syms}) +list(FILTER _tagged INCLUDE REGEX "(^|[^0-9])6jinja22v2") +list(LENGTH _tagged _tagged) +set(_rest ${_syms}) +list(TRANSFORM _rest REPLACE "(^|[^0-9])6jinja22v2" "\\1") +list(FILTER _rest INCLUDE REGEX "(^|[^0-9])6jinja2|^_ZN?K?3xyz") +# report the symbols as nm printed them +set(_bad "") +if (_rest) + foreach (_sym IN LISTS _syms) + string(REGEX REPLACE "(^|[^0-9])6jinja22v2" "\\1" _stripped "${_sym}") + if (_stripped MATCHES "(^|[^0-9])6jinja2|^_ZN?K?3xyz") + list(APPEND _bad "${_sym}") + endif () + endforeach () +endif () +list(REMOVE_DUPLICATES _bad) +list(LENGTH _bad _nbad) +if (_nbad GREATER 0) + list(SUBLIST _bad 0 20 _shown) + string(REPLACE ";" "\n " _shown "${_shown}") + message(FATAL_ERROR "${_nbad} symbols outside jinja2::v2 in ${LIBRARY}, run scripts/abi_namespace.py:\n ${_shown}") +endif () +if (_tagged EQUAL 0) + message(FATAL_ERROR "no jinja2::v2 symbols in ${LIBRARY}: is ${NM} reading the right file?") +endif () +message(STATUS "${_tagged} jinja2::v2 symbols, none outside it") diff --git a/test/containers_api_test.cpp b/test/containers_api_test.cpp index 5f3dda65c..46f24ad62 100644 --- a/test/containers_api_test.cpp +++ b/test/containers_api_test.cpp @@ -166,11 +166,13 @@ struct Reflector static Value CreateFromPtr(const Celsius* val) { return Create(*val); } }; -namespace detail -{ -// A 1.x-style specialisation keeps working +} // namespace jinja2 + +// A 1.x-style specialisation keeps working. It is written with the qualified name: MSVC does +// not implement CWG 2061, so `namespace jinja2 { namespace detail {` there declares a new +// jinja2::detail beside jinja2::v2::detail instead of reopening it (docs/tasks/0076, README) template<> -struct Reflector +struct jinja2::detail::Reflector { static Value Create(const LegacyReflected& val) { return Value(static_cast(val.value * 10)); } static Value CreateFromPtr(const LegacyReflected* val) { return Create(*val); } @@ -178,20 +180,18 @@ struct Reflector // 1.x code could delegate to the library's reflectors, and override one for a type the library also covers template<> -struct Reflector +struct jinja2::detail::Reflector { static Value Create(const LegacyDelegating& val) { return Reflector::Create(val.text + "!"); } static Value CreateFromPtr(const LegacyDelegating* val) { return Create(*val); } }; template<> -struct Reflector +struct jinja2::detail::Reflector { static Value Create(char16_t val) { return Value(std::string(1, static_cast(val))); } static Value CreateFromPtr(const char16_t* val) { return Create(*val); } }; -} // namespace detail -} // namespace jinja2 TEST(ContainersApiTest, ErrorCodeValuesAreStable) { diff --git a/test/filters_test.cpp b/test/filters_test.cpp index e5399913a..8244b33c6 100644 --- a/test/filters_test.cpp +++ b/test/filters_test.cpp @@ -690,6 +690,7 @@ INSTANTIATE_TEST_SUITE_P(EscapeCpp, FilterGenericTest, ::testing::Values( INSTANTIATE_TEST_SUITE_P(ToYaml, FilterGenericTest, ::testing::Values( InputOutputPair{"{'a': 1, 'b': [1, 2], 'c': {'d': none, 'e': 'x y'}} | toyaml", "a: 1\nb:\n - 1\n - 2\nc:\n d: null\n e: x y"}, + InputOutputPair{"{1: 'a', (1, 2): 'b', none: 'c'} | toyaml", "\"1\": a\n\"(1, 2)\": b\nNone: c"}, InputOutputPair{"[1, 'two', [3, 4], {'k': true}, [], {}] | toyaml", "- 1\n- two\n- - 3\n - 4\n- k: true\n- []\n- {}"}, InputOutputPair{"[{'a': 1, 'b': {'c': 2}}] | toyaml", "- a: 1\n b:\n c: 2"}, diff --git a/test/internal_value_test.cpp b/test/internal_value_test.cpp index efe3a86b2..c6983694e 100644 --- a/test/internal_value_test.cpp +++ b/test/internal_value_test.cpp @@ -1,4 +1,5 @@ #include "../src/internal_value.h" +#include "../src/template_dict.h" #include "../src/value_visitors.h" #include "gtest/gtest.h" @@ -10,12 +11,16 @@ #include #include #include +#include +#include #include +#include #include #include #include #include #include +#include using namespace jinja2; @@ -267,4 +272,154 @@ TEST(InternalValueLayoutTest, HostMarkupRoundTrips) EXPECT_TRUE(IntValue2Value(InternalValue(std::string_view(text)).SetMarkup()).IsMarkup()) << size; } } -#endif // JINJA2CPP_LINK_AS_SHARED + +// Dict keys follow Python's equality (docs/tasks/0036): the token is the same for keys that are +// one key +TEST(TemplateDictTest, KeyTokensFollowPythonEquality) +{ + const auto token = [](const InternalValue& key) { return MakeDictKey(key); }; + const auto tuple = [](InternalValueList items) { return InternalValue(ListAdapter::CreateAdapter(std::move(items)).MarkAsTuple()); }; + const InternalValue one(int64_t{ 1 }); + + ASSERT_TRUE(token(one)); + EXPECT_EQ(token(one), token(InternalValue(1.0))); + EXPECT_EQ(token(one), token(InternalValue(true))); + EXPECT_NE(token(one), token(InternalValue(1.5))); + EXPECT_NE(token(one), token(InternalValue(int64_t{ 2 }))); + EXPECT_EQ(token(InternalValue(int64_t{ 0 })), token(InternalValue(-0.0))); + EXPECT_EQ(token(InternalValue(false)), token(InternalValue(0.0))); + // A float beyond int64 is still a float + EXPECT_NE(token(InternalValue(9223372036854775808.0)), token(InternalValue(std::numeric_limits::max()))); + + EXPECT_EQ(token(tuple({ one, InternalValue(2.0) })), token(tuple({ InternalValue(1.0), InternalValue(int64_t{ 2 }) }))); + EXPECT_NE(token(tuple({ one, InternalValue(int64_t{ 2 }) })), token(tuple({ InternalValue(int64_t{ 2 }), one }))); + // (1, 2) is not the nested ((1), 2) nor the longer (1, 2, 3) + EXPECT_NE(token(tuple({ one, InternalValue(int64_t{ 2 }) })), token(tuple({ one, InternalValue(int64_t{ 2 }), one }))); + EXPECT_EQ(token(tuple({ tuple({ one }), InternalValue(std::string("a")) })), token(tuple({ tuple({ InternalValue(1.0) }), InternalValue(std::wstring(L"a")) }))); + + EXPECT_EQ(token(InternalValue(std::string("a"))), token(InternalValue(std::wstring(L"a")))); + EXPECT_EQ(std::optional("a"), token(InternalValue(std::string("a")))); + // A string that spells the token of the integer 1 is another key + const auto oneToken = token(one); + ASSERT_TRUE(oneToken.has_value()); + EXPECT_NE(oneToken, token(InternalValue(oneToken.value_or(std::string())))); + EXPECT_NE(token(InternalValue(NoneValue{})), token(InternalValue(std::string("N")))); + + const auto range = [](int64_t start, int64_t stop, int64_t step) { return InternalValue(ListAdapter::CreateRange(start, stop, step)); }; + // Equal ranges are equal keys: range(0, 3, 2) and range(0, 4, 2) are both [0, 2] + EXPECT_EQ(token(range(0, 3, 2)), token(range(0, 4, 2))); + EXPECT_EQ(token(range(5, 5, 1)), token(range(0, 0, 3))); + EXPECT_NE(token(range(0, 3, 1)), token(range(0, 4, 2))); +} + +TEST(TemplateDictTest, UnhashableKeys) +{ + const char* type = nullptr; + const InternalValue list(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(int64_t{ 1 }) })); + EXPECT_FALSE(MakeDictKey(list, &type)); + EXPECT_STREQ("list", type); + + const InternalValue tupleWithList(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(int64_t{ 1 }), list }).MarkAsTuple()); + type = nullptr; + EXPECT_FALSE(MakeDictKey(tupleWithList, &type)); + EXPECT_STREQ("list", type); + + const InternalValue dict(CreateMapAdapter(TemplateDict())); + type = nullptr; + EXPECT_FALSE(MakeDictKey(dict, &type)); + EXPECT_STREQ("dict", type); + + TemplateDict target; + EXPECT_THROW(target.Set(list, InternalValue(int64_t{ 1 })), std::runtime_error); + EXPECT_TRUE(target.empty()); + EXPECT_EQ(target.end(), target.Find(list)); +} + +TEST(TemplateDictTest, FirstKeyAndPositionKeptLastValueWins) +{ + TemplateDict dict; + dict.Set(InternalValue(int64_t{ 1 }), InternalValue(std::string("a"))); + dict.Set(InternalValue(std::string("x")), InternalValue(std::string("b"))); + dict.Set(InternalValue(1.0), InternalValue(std::string("c"))); + dict.Set(InternalValue(true), InternalValue(std::string("d"))); + + ASSERT_EQ(2U, dict.size()); + EXPECT_TRUE(dict.HasKeyValues()); + auto first = dict.begin(); + EXPECT_EQ(ValueKind::Int, dict.KeyOf(first->first).Kind()); + EXPECT_EQ("d", AsString(first->second)); + EXPECT_EQ(ValueKind::String, dict.KeyOf(std::next(first)->first).Kind()); + + auto found = dict.Find(InternalValue(1.0)); + ASSERT_NE(dict.end(), found); + EXPECT_EQ("d", AsString(found->second)); + dict.Erase(found); + EXPECT_FALSE(dict.HasKeyValues()); + EXPECT_EQ(1U, dict.size()); + + // A copy has keys of its own + dict.Set(InternalValue(2.5), InternalValue(int64_t{ 1 })); + TemplateDict copy(dict); + copy.Set(InternalValue(2.5), InternalValue(int64_t{ 2 })); + copy.Set(InternalValue(3.5), InternalValue(int64_t{ 3 })); + EXPECT_EQ(2U, dict.size()); + EXPECT_EQ(3U, copy.size()); + const auto key = dict.KeyOf(dict.Find(InternalValue(2.5))->first); + ASSERT_NE(nullptr, GetIf(&key)); + EXPECT_EQ(2.5, *GetIf(&key)); + EXPECT_FALSE(dict == copy); +} + +TEST(TemplateDictTest, StringKeysStartingWithTheMarkAreEscaped) +{ + TemplateDict dict; + const std::string odd = std::string(1, DictKeyMark) + "n12345678"; + dict.SetString(odd, InternalValue(int64_t{ 7 })); + dict.Set(InternalValue(int64_t{ 1 }), InternalValue(int64_t{ 8 })); + + ASSERT_EQ(2U, dict.size()); + ASSERT_NE(dict.end(), dict.FindString(odd)); + EXPECT_EQ(7, ConvertToInt(dict.FindString(odd)->second)); + ASSERT_NE(dict.end(), dict.Find(InternalValue(odd))); + // Read back as the string it was + EXPECT_EQ(odd, AsString(dict.KeyOf(dict.begin()->first))); + EXPECT_EQ(ValueKind::Int, dict.KeyOf(std::next(dict.begin())->first).Kind()); +} + +TEST(TemplateDictTest, PublicMapSpellsKeys) +{ + TemplateDict dict; + dict.Set(InternalValue(int64_t{ 1 }), InternalValue(std::string("x"))); + dict.SetString("b", InternalValue(int64_t{ 2 })); + dict.Set(InternalValue(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(int64_t{ 1 }), InternalValue(int64_t{ 2 }) }).MarkAsTuple()), InternalValue(int64_t{ 3 })); + dict.Set(InternalValue(std::string("1")), InternalValue(std::string("y"))); + + const auto map = CreateMapAdapter(std::move(dict)); + const auto value = IntValue2Value(InternalValue(map)); + const auto generic = value.ToGenericMap(); + ASSERT_NE(nullptr, generic.GetAccessor()); + // Later value wins at the position of the first spelling + EXPECT_EQ((std::vector{ "1", "b", "(1, 2)" }), generic.GetKeys()); + EXPECT_EQ("y", generic.GetValueByName("1").ToString()); + EXPECT_EQ(3U, generic.GetSize()); + // A copy: later changes of the template's dict do not reach it + map.GetMutableItems()->SetString("c", InternalValue(int64_t{ 4 })); + EXPECT_EQ(3U, generic.GetSize()); + + // A dict of string keys stays a live view + TemplateDict strings; + strings.SetString("a", InternalValue(int64_t{ 1 })); + const auto stringMap = CreateMapAdapter(std::move(strings)); + const auto stringGeneric = IntValue2Value(InternalValue(stringMap)).ToGenericMap(); + EXPECT_EQ(1U, stringGeneric.GetSize()); + stringMap.GetMutableItems()->SetString("b", InternalValue(int64_t{ 2 })); + EXPECT_EQ(2U, stringGeneric.GetSize()); +} + +TEST(TemplateDictTest, DictsCompareAsPythonDicts) +{ + Template tpl; + ASSERT_TRUE(tpl.Load("{{ {1: 'a', (1, 2): 'b'} == {(1.0, 2): 'b', True: 'a'} }}|{{ {1: 'a'} == {'1': 'a'} }}|{{ 1 in {True: 0} }}").has_value()); + EXPECT_EQ("True|False|True", tpl.RenderAsString({}).value()); +} +#endif diff --git a/test/node_arena_test.cpp b/test/node_arena_test.cpp index b204d31b9..9ec3f8cff 100644 --- a/test/node_arena_test.cpp +++ b/test/node_arena_test.cpp @@ -31,10 +31,10 @@ using namespace jinja2; -namespace jinja2::detail -{ -// Handles from raw offsets and the bytes of a sealed tree, to corrupt a tree on purpose -struct ArenaTestAccess +// Handles from raw offsets and the bytes of a sealed tree, to corrupt a tree on purpose. +// Defined through its qualified name: reopening `namespace jinja2::detail` here would make +// MSVC declare a new jinja2::detail beside jinja2::v2::detail (docs/tasks/0076) +struct jinja2::detail::ArenaTestAccess { template static NodeRef Ref(std::uint32_t offset) @@ -65,7 +65,6 @@ struct ArenaTestAccess return header.objects; } }; -} // namespace jinja2::detail namespace { diff --git a/test/package/CMakeLists.txt b/test/package/CMakeLists.txt index 0a78725e6..e455b2538 100644 --- a/test/package/CMakeLists.txt +++ b/test/package/CMakeLists.txt @@ -18,6 +18,21 @@ enable_testing() add_test(NAME consumer COMMAND consumer ${CMAKE_CURRENT_BINARY_DIR}/templates) add_test(NAME consumer_compat COMMAND consumer_compat ${CMAKE_CURRENT_BINARY_DIR}/templates) +# A program built against 1.x headers must fail to link with this library, not misbehave: +# the inline namespace v2 is in every mangled name (docs/tasks/0076). abi_probe_v2 is the +# positive control; abi_probe_v1 is built only by the test, which expects the link error. +add_executable(abi_probe_v2 abi_probe.cpp) +target_compile_definitions(abi_probe_v2 PRIVATE PROBE_V2) +target_link_libraries(abi_probe_v2 PRIVATE jinja2cpp::jinja2cpp) +add_test(NAME abi_probe_v2 COMMAND abi_probe_v2) +add_executable(abi_probe_v1 EXCLUDE_FROM_ALL abi_probe.cpp) +target_link_libraries(abi_probe_v1 PRIVATE jinja2cpp::jinja2cpp) +add_test(NAME abi_probe_v1_does_not_link + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target abi_probe_v1 --config $) +# GNU ld, lld, Apple ld and MSVC link; a compile error or a successful link does not match +set_tests_properties(abi_probe_v1_does_not_link PROPERTIES PASS_REGULAR_EXPRESSION + "undefined reference to [`']?jinja2::operator==|undefined symbol: jinja2::operator==|Undefined symbols for architecture|unresolved external symbol") + # check_package.cmake points PKG_CONFIG_PATH at the install and turns this on. pkg-config # lists the private dependencies of a static build only in the internal deps mode, where # it knows them. diff --git a/test/package/abi_probe.cpp b/test/package/abi_probe.cpp new file mode 100644 index 000000000..4747abe8b --- /dev/null +++ b/test/package/abi_probe.cpp @@ -0,0 +1,28 @@ +// Negative link check for the inline ABI namespace (docs/tasks/0076). No Jinja2C++ header is +// included: the declaration below is what a program compiled against the 1.x headers refers +// to. Built with PROBE_V2 it names jinja2::v2::operator== and must link and run; without it, +// it names the 1.x jinja2::operator==, which the 2.x library must not define, so the link +// has to fail (test/package/CMakeLists.txt). +#ifdef PROBE_V2 +namespace jinja2 +{ +inline namespace v2 +{ +class Value; +bool operator==(const Value& lhs, const Value& rhs); +} // namespace v2 +} // namespace jinja2 +#else +namespace jinja2 +{ +class Value; +bool operator==(const Value& lhs, const Value& rhs); +} // namespace jinja2 +#endif + +int main() +{ + // volatile keeps the reference to the library symbol in the object file + bool (*volatile compare)(const jinja2::Value&, const jinja2::Value&) = &jinja2::operator==; + return compare ? 0 : 1; +} diff --git a/test/package/check_package.cmake b/test/package/check_package.cmake index 4739b5d07..5b2467329 100644 --- a/test/package/check_package.cmake +++ b/test/package/check_package.cmake @@ -51,6 +51,34 @@ foreach (_req "${_major}.0;TRUE" "${_next}.0;FALSE" "${_prev}.0;FALSE") message(FATAL_ERROR "version file: ${PACKAGE_FIND_VERSION} compatible=${PACKAGE_VERSION_COMPATIBLE}, want ${_want}") endif () endforeach () +# The installed SONAME carries SOVERSION (docs/tasks/0076): the loader refuses a shared +# library of another ABI version, which is what protects shared consumers. SOVERSION tracks +# the ABI, not the major, so it is read from CMakeLists.txt rather than derived from _major. +if (EXPECT_SHARED AND NOT CMAKE_HOST_WIN32) + file(STRINGS "${_src}/../../CMakeLists.txt" _soversion REGEX "^[ \t]*SOVERSION[ \t]+[0-9]+") + string(REGEX MATCH "[0-9]+" _soversion "${_soversion}") + if (NOT _soversion) + message(FATAL_ERROR "no SOVERSION found in ${_src}/../../CMakeLists.txt") + endif () + if (CMAKE_HOST_APPLE) + set(_soname "libjinja2cpp.${_soversion}.dylib") + else () + set(_soname "libjinja2cpp.so.${_soversion}") + endif () + file(GLOB _solib "${PREFIX}/lib/${_soname}" "${PREFIX}/lib64/${_soname}" "${PREFIX}/lib/*/${_soname}") + if (NOT _solib) + file(GLOB_RECURSE _installed RELATIVE "${PREFIX}" "${PREFIX}/*jinja2cpp*") + message(FATAL_ERROR "${_soname} is not installed; found: ${_installed}") + endif () + list(GET _solib 0 _solib) + find_program(_readelf readelf) + if (_readelf AND NOT CMAKE_HOST_APPLE) + execute_process(COMMAND "${_readelf}" -d "${_solib}" OUTPUT_VARIABLE _dynamic) + if (NOT _dynamic MATCHES "SONAME\\)? *Library soname: \\[${_soname}\\]") + message(FATAL_ERROR "SONAME of ${_solib} is not ${_soname}:\n${_dynamic}") + endif () + endif () +endif () set(ENV{PKG_CONFIG_PATH} "") file(GLOB_RECURSE _pcdirs LIST_DIRECTORIES true "${PREFIX}/*/pkgconfig") string(REPLACE ";" ":" _pcpath "${_pcdirs}") diff --git a/test/package/main.cpp b/test/package/main.cpp index 63cac50d2..1a7196d5c 100644 --- a/test/package/main.cpp +++ b/test/package/main.cpp @@ -12,8 +12,9 @@ static_assert(nsel_USES_STD_EXPECTED == 0, "Result must be expected-lite's ow #if defined(EXPECT_SHARED) != defined(JINJA2CPP_LINK_AS_SHARED) #error "JINJA2CPP_LINK_AS_SHARED does not match the installed library's link type" #endif -// The generated version.h is installed with the headers (docs/tasks/0069) -static_assert(JINJA2CPP_VERSION >= 10302, "jinja2cpp/version.h is stale or missing"); +// The generated version.h is installed with the headers (docs/tasks/0069), and its major +// version is the one in the inline ABI namespace v2 (docs/tasks/0076) +static_assert(JINJA2CPP_VERSION_MAJOR == 2, "jinja2cpp/version.h is stale or missing, or the ABI namespace tag needs a new major"); namespace { diff --git a/test/parity/cases/errors.py b/test/parity/cases/errors.py index f8c7e3ea9..ea91bba18 100644 --- a/test/parity/cases/errors.py +++ b/test/parity/cases/errors.py @@ -102,4 +102,10 @@ # caller() outside a call block raises "No caller defined" (docs/tasks/0169) ("caller_without_call_block", "{% macro m() %}[{{ caller() }}]{% endmacro %}{{ m() }}"), ("call_body_reads_own_missing_caller", "{% macro inner() %}<{{ caller() }}>{% endmacro %}{% macro m() %}{% call inner() %}{{ caller() }}{% endcall %}{% endmacro %}{% call m() %}x{% endcall %}"), + ('dict_unhashable_dict_key', "{{ {{'a':1}: 2} }}"), + ('dict_unhashable_tuple_key', '{{ {(1, [2]): 3} }}'), + ('in_dict_unhashable', '{{ [1] in {1: 2} }}'), + ('dict_get_unhashable', '{{ {1: 2}.get([1]) }}'), + ('dict_setdefault_unhashable', '{{ {1: 2}.setdefault([1], 3) }}'), + ('dict_pop_unhashable', '{{ {1: 2}.pop([1]) }}'), ] diff --git a/test/parity/cases/filters.py b/test/parity/cases/filters.py index 83aa22d88..2dd9b41ee 100644 --- a/test/parity/cases/filters.py +++ b/test/parity/cases/filters.py @@ -331,4 +331,23 @@ ("truncate_int", "{{ 5|truncate }}"), ("truncate_undefined_checks_length", "{{ missing|truncate(1, leeway=0) }}"), ("truncate_end_not_string", "{{ 'abc'|truncate(1, leeway=0, end=1) }}"), + ('list_dict_int_keys', "{{ {2: 'a', 1: 'b'}|list }}|{{ {2: 'a', 1.5: 'b', None: 'c'}|list }}|{{ {1: 'a', 2: 'b'}|first }}|{{ {1: 'a', 2: 'b'}|length }}"), + ('dictsort_numeric_keys', "{{ {10: 'a', 2: 'b', 2.5: 'c', True: 'd'}|dictsort }}"), + ('dictsort_mixed_key_types', "{{ {'b': 1, 1: 2}|dictsort }}"), + ('dictsort_none_keys', '{{ {None: 1, 2: 2}|dictsort }}'), + ('dictsort_single_none_key', '{{ {None: 1}|dictsort }}'), + ('dictsort_reverse_numeric', "{{ {1: 'a', 3: 'c', 2: 'b'}|dictsort(reverse=true) }}"), + ('tojson_non_string_keys', "{{ {1: 'a', 2.5: 'b'}|tojson }}|{{ {None: 1}|tojson }}|{{ {True: 1}|tojson }}"), + ('tojson_key_order', "{{ {10: 'a', 2: 'b', 1.5: 'c', False: 'd'}|tojson }}"), + ('tojson_tuple_key', '{{ {(1, 2): 1}|tojson }}'), + ('tojson_mixed_key_types', "{{ {'a': 1, 2: 2}|tojson }}"), + ('tojson_none_among_keys', "{{ {None: 1, 'a': 2}|tojson }}"), + ('xmlattr_int_key', "{{ {1: 'a'}|xmlattr }}"), + ('xmlattr_int_key_none_value', '[{{ {1: none}|xmlattr }}]'), + ('urlencode_int_key', '{{ {1: 2}|urlencode }}'), + ('pprint_non_string_keys', "{{ {10: 2, 2: 1, 'a': 3}|pprint }}|{{ {(1, 2): 3}|pprint }}"), + ('pprint_mixed_type_keys', "{{ {'b': 1, 2.5: 2, None: 3, 1: 4, (1, 2): 5, True: 6}|pprint }}"), + ('string_dict_non_string_keys', "{{ {1: 'a', None: 'b', (1, 2): 'c'}|string }}"), + ('format_percent_int_key', "{{ '%(1)s'|format(a=1) }}"), + ('map_over_dict_int_keys', "{{ {1: 'a', 2: 'b'}|map('string')|join(',') }}|{{ {1: 'a', 2: 'b'}|sum }}"), ] diff --git a/test/parity/cases/globals.py b/test/parity/cases/globals.py index 7c1a1f7d4..7c867d6f3 100644 --- a/test/parity/cases/globals.py +++ b/test/parity/cases/globals.py @@ -77,4 +77,8 @@ ("lipsum_no_paragraphs", "[{{ lipsum(0, min=200) }}]"), # an object can be a dict key: Python hashes it by identity (0212) ("object_dict_key", "{{ {namespace(): 1} }}"), + ('dict_from_non_string_pairs', "{{ dict([(1, 'a'), ((1, 2), 'b')]) }}|{{ dict({1: 'a'}, b=2) }}|{{ namespace({1: 2}) }}"), + ('dict_unhashable_pair_key', '{{ dict([([1], 2)]) }}'), + ('dict_merge_equal_numeric_keys', "{{ dict([(1, 'a'), (1.0, 'b'), (True, 'c')]) }}"), + ('namespace_non_string_subscript', '{% set ns = namespace(a=1) %}[{{ ns[1] }}][{{ ns.a }}]'), ] diff --git a/test/parity/cases/literals.py b/test/parity/cases/literals.py index 9da45a500..4d173d22b 100644 --- a/test/parity/cases/literals.py +++ b/test/parity/cases/literals.py @@ -68,4 +68,11 @@ ("dict_nested_lookup", "{% set d = {'a': {'b': [1, 2]}} %}{{ d['a']['b'][1] }}"), ("parenthesised", "{{ (1) }}"), ("number_dot_attribute", "[{{ 1.e3 }}][{{ 1.5.2 }}]"), + ('dict_int_key_print', "{{ {1: 'x', '1': 'y'} }}"), + ('dict_numeric_keys_merge', "{{ {1:'a', 1.0:'b', True:'c'} }}|{{ {1.0:'x'} }}|{{ {True:1}[1] }}|{{ {0:'z'}[false] }}"), + ('dict_other_hashable_keys', '{{ {none:1, 1.5:2, (1,2):3} }}|{{ {(1,2):3}[(1,2)] }}|{{ {(1,2.0):3}[(1.0,2)] }}|{{ {none:1}[none] }}'), + ('dict_int_key_not_string', "{{ {1:'x'}['1'] is defined }}|{{ {'1':'x'}[1] is defined }}|{{ {1:2}.1 }}|{{ {-1:2}[-1] }}"), + ('dict_float_keys', "{{ {0.1+0.2:'x'} }}|{{ {-0.0:'x', 0:'y'} }}|{{ {1e100:'x'} }}"), + ('dict_key_range_and_nested_tuple', "{{ {range(3): 'r', ((1, 2), 'a'): 'n'} }}|{{ {range(3): 'r'}[range(0, 3)] }}|{{ {((1, 2), 'a'): 'n'}[((1.0, 2), 'a')] }}"), + ('dict_key_big_float_is_float', "{{ {2.0**63: 'f'}[2.0**63] }}|{{ {2.0**63: 'f'} }}"), ] diff --git a/test/parity/cases/methods.py b/test/parity/cases/methods.py index 66fe6c62c..1eb857dfd 100644 --- a/test/parity/cases/methods.py +++ b/test/parity/cases/methods.py @@ -104,4 +104,13 @@ ("str_format_huge_width", "{{ '{:99999999999999999999}'.format(1) }}"), ("loop_cycle_no_args", "{% for x in [1, 2] %}{{ loop.cycle() }}{% endfor %}"), ("loop_attr_not_dict_method", "{% for i in [1] %}{{ loop.items is defined }}{% endfor %}"), + ('dict_methods_non_string_keys', "{% set d = {1: 'a'} %}{{ d.get(1) }}|{{ d.get(1.0) }}|{{ d.get('1', 'none') }}|{{ d.keys()|list }}|{{ d.setdefault(2.0, 'b') }}|{% do d.update({(1, 2): 'c'}) %}{{ d }}|{{ d.pop(1) }}|{{ d.popitem() }}|{{ d }}", {"env": {"extensions": ["do"]}}), + ('dict_get_host_int_key', "{{ dd.get(1) }}|{{ dd.get(1, 'dflt') }}", {"ctx": {"dd": {"1": 2}}}), + ('dict_pop_missing_int', '{{ {}.pop(1) }}'), + ('format_index_int_key', "{{ '{0[1]}'.format({1: 2}) }}"), + ('dict_update_pairs_and_kwargs', "{% set d = {1: 'a'} %}{% do d.update([(1.0, 'b'), (None, 'n')], z=1) %}{{ d }}", {"env": {"extensions": ["do"]}}), + ('dict_copy_keeps_keys', "{% set d = {1: 'a', (2, 3): 'b'} %}{% set c = d.copy() %}{% do c.update({1: 'x'}) %}{{ d }}|{{ c }}", {"env": {"extensions": ["do"]}}), + ('dict_setdefault_existing_float', "{% set d = {1: 'a'} %}{{ d.setdefault(1.0, 'z') }}{{ d }}", {"env": {"extensions": ["do"]}}), + ('dict_update_unhashable_unchanged', "{% set d = {1: 'a'} %}{% do d.update([(2, 'b'), ([3], 'c')]) %}", {"env": {"extensions": ["do"]}}), + ('dict_store_item_non_string', "{% set d = {1: 'a'} %}{% do d.update({2: 'b'}) %}{{ d[2] }}{{ d.get(2) }}{{ d|length }}", {"env": {"extensions": ["do"]}}), ] diff --git a/test/parity/cases/operators.py b/test/parity/cases/operators.py index 636b894e0..157879026 100644 --- a/test/parity/cases/operators.py +++ b/test/parity/cases/operators.py @@ -123,4 +123,8 @@ ("precedence_compare_operands", "{{ 'T' if x + 1 == 2 * 2 > x - 1 else 'F' }}|{{ 'T' if x - 3 not in l and 2 in l else 'F' }}"), ("precedence_unary_filter", "{{ -x|abs + 1 }}|{{ - - x }}|{{ 2 ** -x|abs }}"), ("not_after_binary_operator", "{{ x + not z }}"), + ('in_dict_non_string_key', "{{ 'T' if 1 in {1: 'a'} else 'F' }}|{{ 'T' if '1' in {1: 'a'} else 'F' }}|{{ 'T' if 1.0 in {1: 'a'} else 'F' }}|{{ 'T' if (1, 2) in {(1, 2): 'a'} else 'F' }}|{{ 'T' if none in {none: 'a'} else 'F' }}"), + ('eq_dict_non_string_keys', "{{ 'T' if {1: 'a'} == {1.0: 'a'} else 'F' }}|{{ 'T' if {1: 'a'} == {'1': 'a'} else 'F' }}|{{ 'T' if {(1, 2): 3} == {(1.0, 2): 3} else 'F' }}"), + ('in_dict_object_key', "{{ 'T' if nothing in {'': 1} else 'F' }}|{{ 'T' if namespace() in {'': 1} else 'F' }}"), + ('not_in_dict_non_string_key', "{{ 'T' if 2 not in {1: 'a'} else 'F' }}|{{ 'T' if True in {1: 'a'} else 'F' }}"), ] diff --git a/test/parity/cases/statements.py b/test/parity/cases/statements.py index 0bdc08914..3fab3fef8 100644 --- a/test/parity/cases/statements.py +++ b/test/parity/cases/statements.py @@ -96,6 +96,8 @@ ("namespace_missing_attr", "{% set ns = namespace() %}[{{ ns.a }}]"), ("namespace_contains_itself", "{% set ns = namespace(a=1) %}{% set ns.a = ns %}{{ ns }}|{{ 'T' if ns.a.a.a is sameas ns else 'F' }}"), ("namespace_contains_itself_in_list", "{% set ns = namespace() %}{% set ns.l = [ns] %}{{ ns }}"), + ("namespace_contains_itself_through_key", "{% set ns = namespace() %}{% set d = {(ns, 1): 1} %}{% set ns.d = d %}{{ d|length }}"), + ("dict_contains_itself_through_setdefault_key", "{% set ns = namespace() %}{% set d = {} %}{% set ns.d = d %}{{ d.setdefault((ns, 1), 2) }}|{{ d|length }}"), ("namespace_set_on_dict", "{% set m = {} %}{% set m.a = 1 %}"), ("namespace_set_on_undefined", "{% set nope.a = 1 %}"), ("namespace_keeps_loop", "{% set ns = namespace() %}{% for i in l %}{% set ns.x = loop %}{% endfor %}{{ ns.x.index }}{{ ns.x.length }}{{ ns.x.revindex }}{{ 'T' if ns.x.last else 'F' }}"), @@ -437,4 +439,6 @@ # the else frame binds at entry, and its set hides a loop target outside only there (docs/tasks/0202) ('idtracking_loop_else_binds_at_entry', '{% for i in e %}{% else %}{% for k in [1] %}{{ x }}{% endfor %}{% set x = 1 %}{% endfor %}'), ('loop_else_set_keeps_outer_target', '{% for x in [1] %}{% for j in e %}{% else %}{% set x = 2 %}{{ x }}{% endfor %}{{ x }}{% endfor %}'), + ('for_dict_int_keys', "{% for k in {1: 'a', 2: 'b'} %}{{ k + 1 }}{% endfor %}|{% for k, v in {2: 'a'}.items() %}{{ k }}{{ v }}{% endfor %}|{% for k, v in {20: 'x', 10: 'y'}|dictsort %}{{ k }}{% endfor %}"), + ('for_dict_tuple_key_unpack', "{% for k, v in {(1, 2): 'a'}.items() %}{{ k[0] }}{{ k[1] }}{{ v }}{% endfor %}"), ] diff --git a/test/parity/cases/subscripts.py b/test/parity/cases/subscripts.py index 6949a7eab..e3ace2eb1 100644 --- a/test/parity/cases/subscripts.py +++ b/test/parity/cases/subscripts.py @@ -41,4 +41,7 @@ ("dot_index_chain", "{{ d.b.c }}{{ users.1.name }}"), ("slice_extreme_step", "{{ l[1::9223372036854775807] }}|{{ l[::-9223372036854775807-1] }}"), ("tuple_index", "[{{ l[1, 2] }}][{{ l[] }}]"), + ('dict_unhashable_subscript', '[{{ {1:2}[[1]] }}]'), + ('dict_subscript_float_and_bool', "{{ {1: 'a'}[1.0] }}|{{ {1: 'a'}[true] }}|{{ {1.5: 'f'}[1.5] }}"), + ('dict_subscript_missing_non_string', "[{{ {1: 'a'}[2] }}][{{ {1: 'a'}[none] }}]"), ] diff --git a/test/parity/divergences/errors.txt b/test/parity/divergences/errors.txt index d869f3e9b..cc82b4d1c 100644 --- a/test/parity/divergences/errors.txt +++ b/test/parity/divergences/errors.txt @@ -1,7 +1,6 @@ # Known divergences of Jinja2C++ from Python Jinja2 in the errors area (cases/errors.py). # Format and kinds: test/parity/README.md. -errors.dict_unhashable_key accepts 0036 unhashable dict keys are accepted (keys are stringified) errors.if_test_conditional_expression accepts 0213 an if test takes a conditional expression; Jinja2 parses it without one errors.raw_unclosed_at_end rejects 0213 an unclosed raw block at the end of the template; Jinja2 renders it as empty errors.raw_unclosed_minus_space rejects 0213 an unclosed raw block at the end of the template; Jinja2 renders it as empty diff --git a/test/parity/divergences/literals.txt b/test/parity/divergences/literals.txt index 936bb63c5..6551c7b3d 100644 --- a/test/parity/divergences/literals.txt +++ b/test/parity/divergences/literals.txt @@ -1,6 +1,5 @@ # Known divergences of Jinja2C++ from Python Jinja2 in the literals area (cases/literals.py). # Format and kinds: test/parity/README.md. -literals.dict_int_key output 0036 mapping keys are strings; an integer key cannot be looked up by integer literals.int_big output 0015 deliberate: integers are 64-bit; a literal beyond int64 becomes a float (docs/parity.md) literals.string_escape_named rejects 0041 \N{name} needs the Unicode name table; it is rejected as malformed (documented divergence) diff --git a/test/parity/expected/errors.json b/test/parity/expected/errors.json index 7b917cc32..60ad7370f 100644 --- a/test/parity/expected/errors.json +++ b/test/parity/expected/errors.json @@ -1843,6 +1843,120 @@ "type": "UndefinedError", "message": "No caller defined" } + }, + { + "id": "errors.dict_unhashable_dict_key", + "template": "{{ {{'a':1}: 2} }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'dict'" + } + }, + { + "id": "errors.dict_unhashable_tuple_key", + "template": "{{ {(1, [2]): 3} }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.in_dict_unhashable", + "template": "{{ [1] in {1: 2} }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.dict_get_unhashable", + "template": "{{ {1: 2}.get([1]) }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.dict_setdefault_unhashable", + "template": "{{ {1: 2}.setdefault([1], 3) }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.dict_pop_unhashable", + "template": "{{ {1: 2}.pop([1]) }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } } ] } diff --git a/test/parity/expected/filters.json b/test/parity/expected/filters.json index a389e8a46..6bfb8a73b 100644 --- a/test/parity/expected/filters.json +++ b/test/parity/expected/filters.json @@ -20589,6 +20589,1288 @@ "type": "TypeError", "message": "object of type 'int' has no len()" } + }, + { + "id": "filters.list_dict_int_keys", + "template": "{{ {2: 'a', 1: 'b'}|list }}|{{ {2: 'a', 1.5: 'b', None: 'c'}|list }}|{{ {1: 'a', 2: 'b'}|first }}|{{ {1: 'a', 2: 'b'}|length }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[2, 1]|[2, 1.5, None]|1|2" + }, + { + "id": "filters.dictsort_numeric_keys", + "template": "{{ {10: 'a', 2: 'b', 2.5: 'c', True: 'd'}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[(True, 'd'), (2, 'b'), (2.5, 'c'), (10, 'a')]" + }, + { + "id": "filters.dictsort_mixed_key_types", + "template": "{{ {'b': 1, 1: 2}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'int' and 'str'" + } + }, + { + "id": "filters.dictsort_none_keys", + "template": "{{ {None: 1, 2: 2}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'int' and 'NoneType'" + } + }, + { + "id": "filters.dictsort_single_none_key", + "template": "{{ {None: 1}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[(None, 1)]" + }, + { + "id": "filters.dictsort_reverse_numeric", + "template": "{{ {1: 'a', 3: 'c', 2: 'b'}|dictsort(reverse=true) }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[(3, 'c'), (2, 'b'), (1, 'a')]" + }, + { + "id": "filters.tojson_non_string_keys", + "template": "{{ {1: 'a', 2.5: 'b'}|tojson }}|{{ {None: 1}|tojson }}|{{ {True: 1}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{\"1\": \"a\", \"2.5\": \"b\"}|{\"null\": 1}|{\"true\": 1}" + }, + { + "id": "filters.tojson_key_order", + "template": "{{ {10: 'a', 2: 'b', 1.5: 'c', False: 'd'}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{\"false\": \"d\", \"1.5\": \"c\", \"2\": \"b\", \"10\": \"a\"}" + }, + { + "id": "filters.tojson_tuple_key", + "template": "{{ {(1, 2): 1}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "keys must be str, int, float, bool or None, not tuple" + } + }, + { + "id": "filters.tojson_mixed_key_types", + "template": "{{ {'a': 1, 2: 2}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'int' and 'str'" + } + }, + { + "id": "filters.tojson_none_among_keys", + "template": "{{ {None: 1, 'a': 2}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'str' and 'NoneType'" + } + }, + { + "id": "filters.xmlattr_int_key", + "template": "{{ {1: 'a'}|xmlattr }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "expected string or bytes-like object, got 'int'" + } + }, + { + "id": "filters.xmlattr_int_key_none_value", + "template": "[{{ {1: none}|xmlattr }}]", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[]" + }, + { + "id": "filters.urlencode_int_key", + "template": "{{ {1: 2}|urlencode }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "1=2" + }, + { + "id": "filters.pprint_non_string_keys", + "template": "{{ {10: 2, 2: 1, 'a': 3}|pprint }}|{{ {(1, 2): 3}|pprint }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{2: 1, 10: 2, 'a': 3}|{(1, 2): 3}" + }, + { + "id": "filters.pprint_mixed_type_keys", + "template": "{{ {'b': 1, 2.5: 2, None: 3, 1: 4, (1, 2): 5, True: 6}|pprint }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{None: 3, 1: 6, 2.5: 2, 'b': 1, (1, 2): 5}" + }, + { + "id": "filters.string_dict_non_string_keys", + "template": "{{ {1: 'a', None: 'b', (1, 2): 'c'}|string }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{1: 'a', None: 'b', (1, 2): 'c'}" + }, + { + "id": "filters.format_percent_int_key", + "template": "{{ '%(1)s'|format(a=1) }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "KeyError", + "message": "'1'" + } + }, + { + "id": "filters.map_over_dict_int_keys", + "template": "{{ {1: 'a', 2: 'b'}|map('string')|join(',') }}|{{ {1: 'a', 2: 'b'}|sum }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "1,2|3" } ] } diff --git a/test/parity/expected/globals.json b/test/parity/expected/globals.json index a9d6ba757..385a9433a 100644 --- a/test/parity/expected/globals.json +++ b/test/parity/expected/globals.json @@ -705,6 +705,42 @@ "env": {}, "templates": {}, "output": "{: 1}" + }, + { + "id": "globals.dict_from_non_string_pairs", + "template": "{{ dict([(1, 'a'), ((1, 2), 'b')]) }}|{{ dict({1: 'a'}, b=2) }}|{{ namespace({1: 2}) }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'a', (1, 2): 'b'}|{1: 'a', 'b': 2}|" + }, + { + "id": "globals.dict_unhashable_pair_key", + "template": "{{ dict([([1], 2)]) }}", + "context": {}, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "globals.dict_merge_equal_numeric_keys", + "template": "{{ dict([(1, 'a'), (1.0, 'b'), (True, 'c')]) }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'c'}" + }, + { + "id": "globals.namespace_non_string_subscript", + "template": "{% set ns = namespace(a=1) %}[{{ ns[1] }}][{{ ns.a }}]", + "context": {}, + "env": {}, + "templates": {}, + "output": "[][1]" } ] } diff --git a/test/parity/expected/literals.json b/test/parity/expected/literals.json index 7d5a0c034..fd5c9ed2b 100644 --- a/test/parity/expected/literals.json +++ b/test/parity/expected/literals.json @@ -569,6 +569,62 @@ "env": {}, "templates": {}, "output": "[][]" + }, + { + "id": "literals.dict_int_key_print", + "template": "{{ {1: 'x', '1': 'y'} }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'x', '1': 'y'}" + }, + { + "id": "literals.dict_numeric_keys_merge", + "template": "{{ {1:'a', 1.0:'b', True:'c'} }}|{{ {1.0:'x'} }}|{{ {True:1}[1] }}|{{ {0:'z'}[false] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'c'}|{1.0: 'x'}|1|z" + }, + { + "id": "literals.dict_other_hashable_keys", + "template": "{{ {none:1, 1.5:2, (1,2):3} }}|{{ {(1,2):3}[(1,2)] }}|{{ {(1,2.0):3}[(1.0,2)] }}|{{ {none:1}[none] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{None: 1, 1.5: 2, (1, 2): 3}|3|3|1" + }, + { + "id": "literals.dict_int_key_not_string", + "template": "{{ {1:'x'}['1'] is defined }}|{{ {'1':'x'}[1] is defined }}|{{ {1:2}.1 }}|{{ {-1:2}[-1] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "False|False|2|2" + }, + { + "id": "literals.dict_float_keys", + "template": "{{ {0.1+0.2:'x'} }}|{{ {-0.0:'x', 0:'y'} }}|{{ {1e100:'x'} }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{0.30000000000000004: 'x'}|{-0.0: 'y'}|{1e+100: 'x'}" + }, + { + "id": "literals.dict_key_range_and_nested_tuple", + "template": "{{ {range(3): 'r', ((1, 2), 'a'): 'n'} }}|{{ {range(3): 'r'}[range(0, 3)] }}|{{ {((1, 2), 'a'): 'n'}[((1.0, 2), 'a')] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{range(0, 3): 'r', ((1, 2), 'a'): 'n'}|r|n" + }, + { + "id": "literals.dict_key_big_float_is_float", + "template": "{{ {2.0**63: 'f'}[2.0**63] }}|{{ {2.0**63: 'f'} }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "f|{9.223372036854776e+18: 'f'}" } ] } diff --git a/test/parity/expected/methods.json b/test/parity/expected/methods.json index 976cf5dd7..345a84903 100644 --- a/test/parity/expected/methods.json +++ b/test/parity/expected/methods.json @@ -2129,6 +2129,218 @@ "env": {}, "templates": {}, "output": "False" + }, + { + "id": "methods.dict_methods_non_string_keys", + "template": "{% set d = {1: 'a'} %}{{ d.get(1) }}|{{ d.get(1.0) }}|{{ d.get('1', 'none') }}|{{ d.keys()|list }}|{{ d.setdefault(2.0, 'b') }}|{% do d.update({(1, 2): 'c'}) %}{{ d }}|{{ d.pop(1) }}|{{ d.popitem() }}|{{ d }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "a|a|none|[1]|b|{1: 'a', 2.0: 'b', (1, 2): 'c'}|a|((1, 2), 'c')|{2.0: 'b'}" + }, + { + "id": "methods.dict_get_host_int_key", + "template": "{{ dd.get(1) }}|{{ dd.get(1, 'dflt') }}", + "context": { + "dd": { + "1": 2 + } + }, + "env": {}, + "templates": {}, + "output": "None|dflt" + }, + { + "id": "methods.dict_pop_missing_int", + "template": "{{ {}.pop(1) }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "KeyError", + "message": "1" + } + }, + { + "id": "methods.format_index_int_key", + "template": "{{ '{0[1]}'.format({1: 2}) }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": {}, + "templates": {}, + "output": "2" + }, + { + "id": "methods.dict_update_pairs_and_kwargs", + "template": "{% set d = {1: 'a'} %}{% do d.update([(1.0, 'b'), (None, 'n')], z=1) %}{{ d }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "{1: 'b', None: 'n', 'z': 1}" + }, + { + "id": "methods.dict_copy_keeps_keys", + "template": "{% set d = {1: 'a', (2, 3): 'b'} %}{% set c = d.copy() %}{% do c.update({1: 'x'}) %}{{ d }}|{{ c }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "{1: 'a', (2, 3): 'b'}|{1: 'x', (2, 3): 'b'}" + }, + { + "id": "methods.dict_setdefault_existing_float", + "template": "{% set d = {1: 'a'} %}{{ d.setdefault(1.0, 'z') }}{{ d }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "a{1: 'a'}" + }, + { + "id": "methods.dict_update_unhashable_unchanged", + "template": "{% set d = {1: 'a'} %}{% do d.update([(2, 'b'), ([3], 'c')]) %}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "methods.dict_store_item_non_string", + "template": "{% set d = {1: 'a'} %}{% do d.update({2: 'b'}) %}{{ d[2] }}{{ d.get(2) }}{{ d|length }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "bb2" } ] } diff --git a/test/parity/expected/operators.json b/test/parity/expected/operators.json index 17c396291..110db4e2b 100644 --- a/test/parity/expected/operators.json +++ b/test/parity/expected/operators.json @@ -2653,6 +2653,94 @@ "type": "TemplateSyntaxError", "message": "expected token 'end of print statement', got 'z'" } + }, + { + "id": "operators.in_dict_non_string_key", + "template": "{{ 'T' if 1 in {1: 'a'} else 'F' }}|{{ 'T' if '1' in {1: 'a'} else 'F' }}|{{ 'T' if 1.0 in {1: 'a'} else 'F' }}|{{ 'T' if (1, 2) in {(1, 2): 'a'} else 'F' }}|{{ 'T' if none in {none: 'a'} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "T|F|T|T|T" + }, + { + "id": "operators.eq_dict_non_string_keys", + "template": "{{ 'T' if {1: 'a'} == {1.0: 'a'} else 'F' }}|{{ 'T' if {1: 'a'} == {'1': 'a'} else 'F' }}|{{ 'T' if {(1, 2): 3} == {(1.0, 2): 3} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "T|F|T" + }, + { + "id": "operators.in_dict_object_key", + "template": "{{ 'T' if nothing in {'': 1} else 'F' }}|{{ 'T' if namespace() in {'': 1} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "F|F" + }, + { + "id": "operators.not_in_dict_non_string_key", + "template": "{{ 'T' if 2 not in {1: 'a'} else 'F' }}|{{ 'T' if True in {1: 'a'} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "T|T" } ] } diff --git a/test/parity/expected/statements.json b/test/parity/expected/statements.json index a14824cba..db88ec129 100644 --- a/test/parity/expected/statements.json +++ b/test/parity/expected/statements.json @@ -3498,6 +3498,86 @@ "templates": {}, "output": "]}>" }, + { + "id": "statements.namespace_contains_itself_through_key", + "template": "{% set ns = namespace() %}{% set d = {(ns, 1): 1} %}{% set ns.d = d %}{{ d|length }}", + "context": { + "l": [ + 3, + 1, + 2 + ], + "e": [], + "d": { + "a": 1, + "b": 2 + }, + "x": 3, + "tree": [ + { + "n": "a", + "c": [ + { + "n": "b", + "c": [] + }, + { + "n": "c", + "c": [ + { + "n": "d", + "c": [] + } + ] + } + ] + } + ] + }, + "env": {}, + "templates": {}, + "output": "1" + }, + { + "id": "statements.dict_contains_itself_through_setdefault_key", + "template": "{% set ns = namespace() %}{% set d = {} %}{% set ns.d = d %}{{ d.setdefault((ns, 1), 2) }}|{{ d|length }}", + "context": { + "l": [ + 3, + 1, + 2 + ], + "e": [], + "d": { + "a": 1, + "b": 2 + }, + "x": 3, + "tree": [ + { + "n": "a", + "c": [ + { + "n": "b", + "c": [] + }, + { + "n": "c", + "c": [ + { + "n": "d", + "c": [] + } + ] + } + ] + } + ] + }, + "env": {}, + "templates": {}, + "output": "2|1" + }, { "id": "statements.namespace_set_on_dict", "template": "{% set m = {} %}{% set m.a = 1 %}", @@ -16120,6 +16200,86 @@ "env": {}, "templates": {}, "output": "21" + }, + { + "id": "statements.for_dict_int_keys", + "template": "{% for k in {1: 'a', 2: 'b'} %}{{ k + 1 }}{% endfor %}|{% for k, v in {2: 'a'}.items() %}{{ k }}{{ v }}{% endfor %}|{% for k, v in {20: 'x', 10: 'y'}|dictsort %}{{ k }}{% endfor %}", + "context": { + "l": [ + 3, + 1, + 2 + ], + "e": [], + "d": { + "a": 1, + "b": 2 + }, + "x": 3, + "tree": [ + { + "n": "a", + "c": [ + { + "n": "b", + "c": [] + }, + { + "n": "c", + "c": [ + { + "n": "d", + "c": [] + } + ] + } + ] + } + ] + }, + "env": {}, + "templates": {}, + "output": "23|2a|1020" + }, + { + "id": "statements.for_dict_tuple_key_unpack", + "template": "{% for k, v in {(1, 2): 'a'}.items() %}{{ k[0] }}{{ k[1] }}{{ v }}{% endfor %}", + "context": { + "l": [ + 3, + 1, + 2 + ], + "e": [], + "d": { + "a": 1, + "b": 2 + }, + "x": 3, + "tree": [ + { + "n": "a", + "c": [ + { + "n": "b", + "c": [] + }, + { + "n": "c", + "c": [ + { + "n": "d", + "c": [] + } + ] + } + ] + } + ] + }, + "env": {}, + "templates": {}, + "output": "12a" } ] } diff --git a/test/parity/expected/subscripts.json b/test/parity/expected/subscripts.json index 56fa4276f..6ec8e94b7 100644 --- a/test/parity/expected/subscripts.json +++ b/test/parity/expected/subscripts.json @@ -1239,6 +1239,102 @@ "env": {}, "templates": {}, "output": "[][]" + }, + { + "id": "subscripts.dict_unhashable_subscript", + "template": "[{{ {1:2}[[1]] }}]", + "context": { + "l": [ + 3, + 1, + 2, + 5 + ], + "s": "Hello World", + "d": { + "a": 1, + "b": { + "c": 2 + }, + "key with space": 3, + "items": "own" + }, + "users": [ + { + "name": "bob" + }, + { + "name": "alice" + } + ] + }, + "env": {}, + "templates": {}, + "output": "[]" + }, + { + "id": "subscripts.dict_subscript_float_and_bool", + "template": "{{ {1: 'a'}[1.0] }}|{{ {1: 'a'}[true] }}|{{ {1.5: 'f'}[1.5] }}", + "context": { + "l": [ + 3, + 1, + 2, + 5 + ], + "s": "Hello World", + "d": { + "a": 1, + "b": { + "c": 2 + }, + "key with space": 3, + "items": "own" + }, + "users": [ + { + "name": "bob" + }, + { + "name": "alice" + } + ] + }, + "env": {}, + "templates": {}, + "output": "a|a|f" + }, + { + "id": "subscripts.dict_subscript_missing_non_string", + "template": "[{{ {1: 'a'}[2] }}][{{ {1: 'a'}[none] }}]", + "context": { + "l": [ + 3, + 1, + 2, + 5 + ], + "s": "Hello World", + "d": { + "a": 1, + "b": { + "c": 2 + }, + "key with space": 3, + "items": "own" + }, + "users": [ + { + "name": "bob" + }, + { + "name": "alice" + } + ] + }, + "env": {}, + "templates": {}, + "output": "[][]" } ] } diff --git a/test/recursion_limits_test.cpp b/test/recursion_limits_test.cpp index 9871cf51d..3b30995d3 100644 --- a/test/recursion_limits_test.cpp +++ b/test/recursion_limits_test.cpp @@ -17,6 +17,14 @@ #include #endif +#ifdef __SANITIZE_THREAD__ +#define JINJA2CPP_TEST_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define JINJA2CPP_TEST_TSAN 1 +#endif +#endif + using namespace jinja2; namespace @@ -113,6 +121,40 @@ TEST_F(RecursionLimitsTest, ErrorMessage) EXPECT_NE(std::string::npos, ErrorToString(result.error()).find("error: Maximum recursion depth exceeded\n")) << ErrorToString(result.error()); } +TEST_F(RecursionLimitsTest, DeepTupleKeyRendersOrIsRenderError) +{ + // A dict key is hashed through its nested tuples, as deep as the template made them. The + // stack an optimised build needs for that depth may fit: then the key works as in Python +#ifdef JINJA2CPP_TEST_TSAN + // TSan records every call on a fixed-size shadow stack, which this walk overruns from + // about 16000 levels, before the real stack runs low: __tsan_func_entry then writes past + // it and the run crashes inside TSan. ASan and the plain builds take the full depth + constexpr char depth[] = "10000"; +#else + constexpr char depth[] = "20000"; +#endif + const std::string build = std::string("{% set ns = namespace(t=(1,)) %}{% for i in range(") + depth + ") %}{% set ns.t = (ns.t,) %}{% endfor %}"; + const std::pair cases[] = { + { "{{ {ns.t: 1}|length }}", "1" }, + { "{{ ns.t in {1: 1} }}", "False" }, + { "{{ {1: 1}.get(ns.t) }}", "None" }, + }; + for (const auto& [use, expected] : cases) + { + Template tpl(&m_env); + ASSERT_TRUE(tpl.Load(build + use).has_value()) << use; + auto result = tpl.RenderAsString({}); + if (result.has_value()) + { + EXPECT_EQ(expected, result.value()) << use; + } + else + { + EXPECT_EQ(ErrorCode::RecursionLimitExceeded, result.error().GetCode()) << use; + } + } +} + TEST_F(RecursionLimitsTest, InfiniteRecursionIsRenderError) { AddFile("include_self", "{% include 'include_self' %}"); @@ -278,6 +320,9 @@ TEST_F(RecursionLimitsTest, HugeSequencesFailFast) "{{ ('a\n' * 1000)|indent(2**30) }}", "{{ range(2**40)|reverse|first }}", "{{ ['x']|tojson(2**40) }}", + "{{ {('x',) * 2**40: 1} }}", + "{{ {1: 1}.get(('x',) * 1000000000) }}", + "{{ ('x' * 2**25,) * 4 in {} }}", }; for (const auto& source : templates) { @@ -354,13 +399,6 @@ TEST_F(RecursionLimitsTest, SmallThreadStack) // ==) still recurse, but stop with RecursionLimitExceeded like Python's RecursionError. namespace { -#ifdef __SANITIZE_THREAD__ -#define JINJA2CPP_TEST_TSAN 1 -#elif defined(__has_feature) -#if __has_feature(thread_sanitizer) -#define JINJA2CPP_TEST_TSAN 1 -#endif -#endif #ifdef JINJA2CPP_TEST_TSAN // Deep enough to overflow the walks' thread under TSan too. Deeper, the access history of // the whole run makes LLVM 18's TSan reset its traces, which fails its own CHECK diff --git a/test/template_api_test.cpp b/test/template_api_test.cpp index 666a2bf9e..af1c4ccb6 100644 --- a/test/template_api_test.cpp +++ b/test/template_api_test.cpp @@ -338,7 +338,8 @@ TEST(TemplateApiTest, ConcurrentRenderOfOneTemplate) TEST(TemplateApiTest, VersionMacrosMatchProjectVersion) { static_assert(JINJA2CPP_VERSION == (JINJA2CPP_VERSION_MAJOR * 10000) + (JINJA2CPP_VERSION_MINOR * 100) + JINJA2CPP_VERSION_PATCH); - static_assert(JINJA2CPP_VERSION >= 10302); + // the inline ABI namespace is v2 (docs/tasks/0076); a new major retags it with scripts/abi_namespace.py + static_assert(JINJA2CPP_VERSION_MAJOR == 2); const auto expected = std::to_string(JINJA2CPP_VERSION_MAJOR) + "." + std::to_string(JINJA2CPP_VERSION_MINOR) + "." + std::to_string(JINJA2CPP_VERSION_PATCH); EXPECT_EQ(expected, JINJA2CPP_VERSION_STRING); }