From d6600f076cffb31141c0310d3e18e8f82b73dee3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 11:25:28 +0000 Subject: [PATCH 01/18] Map keys as template values (0036 step 1, no behaviour change) KeyValuePair::key is an InternalValue; IMapAccessor gains GetKeyValues and FindItem; MapRef/MapAdapter::GetKeys are gone. Every internal key reader (repr, ==, list of a map, keys/values/items/copy/update, dictsort, pprint, tojson, xmlattr, urlencode, dict(), pair targets) reads GetEntries or GetKeyValues, with string fast paths kept. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- src/filters.cpp | 50 ++++++++++++++++++++++++------ src/global_functions.cpp | 6 ++-- src/internal_value.cpp | 39 +++++++++++++++++++---- src/internal_value.h | 33 ++++++++++++++++++-- src/serialize_filters.cpp | 55 ++++++++++++++++++++++++--------- src/statements.cpp | 6 ++-- src/string_converter_filter.cpp | 4 +-- src/value_methods.cpp | 44 +++++++------------------- src/value_visitors.h | 16 +++++----- 9 files changed, 171 insertions(+), 82 deletions(-) diff --git a/src/filters.cpp b/src/filters.cpp index 5867861eb..6a807373c 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -467,21 +467,32 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont struct SortItem { KeyValuePair* pair; + // The key as UTF-8 text (lowered unless case-sensitive) when every key is a string std::string sortKey; + // The key otherwise; Python lowers only a str key, not the elements of a tuple + InternalValue sortValue; }; bool (*comparator)(const SortItem& left, const SortItem& right) = nullptr; bool lowerKeys = false; + bool sortByKey = false; + bool keysAreStrings = true; + + auto entries = map->GetEntries(); if (AsString(byVal) == "key") // Sort by key { + sortByKey = true; lowerKeys = !ConvertToBool(isCsVal); - if (lowerKeys) + keysAreStrings = std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); }); + if (keysAreStrings) { comparator = [](const SortItem& left, const SortItem& right) { return left.sortKey < right.sortKey; }; } else { - comparator = [](const SortItem& left, const SortItem& right) { return left.pair->key < right.pair->key; }; + comparator = [](const SortItem& left, const SortItem& right) { + return CompareForOrder(left.sortValue, right.sortValue, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive); + }; } } else if (AsString(byVal) == "value") @@ -505,15 +516,34 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont throw std::runtime_error(R"(You can only sort by either "key" or "value")"); } - auto entries = map->GetEntries(); std::vector tempVector; tempVector.reserve(entries.size()); for (auto& entry : entries) { - SortItem item{ &entry, std::string() }; - if (lowerKeys) + SortItem item{ &entry, std::string(), InternalValue() }; + if (sortByKey && keysAreStrings) { - detail::LowerUtf8(entry.key, item.sortKey); + const auto text = AsString(entry.key); + if (lowerKeys) + { + detail::LowerUtf8(text, item.sortKey); + } + else + { + item.sortKey = text; + } + } + else if (sortByKey) + { + if (lowerKeys && IsStringValue(entry.key)) + { + detail::LowerUtf8(AsString(entry.key), item.sortKey); + item.sortValue = InternalValue(item.sortKey); + } + else + { + item.sortValue = entry.key; + } } tempVector.push_back(std::move(item)); } @@ -1680,7 +1710,7 @@ struct ValueConverterImpl : visitors::BaseVisitor<> return InternalValue(); } - return ListAdapter::CreateAdapter(InternalValueList{ InternalValue(val.key), val.value }); + return ListAdapter::CreateAdapter(InternalValueList{ val.key, val.value }); } InternalValue operator()(const MapRef& val) const @@ -1694,9 +1724,9 @@ struct ValueConverterImpl : visitors::BaseVisitor<> throw std::runtime_error("'"s + objectClass + "' object is not iterable"); } - auto keys = val.GetKeys(); + auto keys = val.GetKeyValues(); auto numKeys = keys.size(); - return ListAdapter::CreateAdapter(numKeys, [values = std::move(keys)](size_t idx) { return InternalValue(values[idx]); }); + return ListAdapter::CreateAdapter(numKeys, [values = std::move(keys)](size_t idx) { return values[idx]; }); } template @@ -2350,7 +2380,7 @@ InternalValue ValueConverter::Items(const InternalValue& baseVal, RenderContext& InternalValueList items; for (auto& [key, value] : map->GetEntries()) { - items.emplace_back(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(std::move(key)), std::move(value) }).MarkAsTuple()); + items.emplace_back(ListAdapter::CreateAdapter(InternalValueList{ std::move(key), std::move(value) }).MarkAsTuple()); } InternalValue result = ListAdapter::CreateAdapter(std::move(items)); if (baseVal.ShouldExtendLifetime()) diff --git a/src/global_functions.cpp b/src/global_functions.cpp index 573a1a83c..58d5a9c27 100644 --- a/src/global_functions.cpp +++ b/src/global_functions.cpp @@ -159,9 +159,9 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) } if (map) { - for (auto& key : map->GetKeys()) + for (auto& [key, value] : map->GetEntries()) { - result[key] = map->GetValueByName(key); + result[KeyToString(key)] = std::move(value); } } else if (auto list = AsList(source)) @@ -170,7 +170,7 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) { if (const auto* pair = GetIf(&item)) { - result[pair->key] = pair->value; + result[KeyToString(pair->key)] = pair->value; continue; } // Any iterable of two items is a pair, a two-character string included diff --git a/src/internal_value.cpp b/src/internal_value.cpp index 5c72ad284..47daf8296 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -890,9 +890,9 @@ struct ListConverter : public visitors::BaseVisitor> } InternalValueList list; - for (auto& k : map.GetKeys()) + for (auto& k : map.GetKeyValues()) { - list.emplace_back(TargetString(k)); + list.emplace_back(std::move(k)); } return ListAdapter::CreateAdapter(std::move(list)); @@ -1662,12 +1662,39 @@ std::vector IMapAccessor::GetEntries() const for (auto& key : keys) { auto value = GetItem(key); - result.push_back(KeyValuePair{ std::move(key), std::move(value) }); + result.push_back(KeyValuePair{ InternalValue(key), std::move(value) }); } return result; } +InternalValueList IMapAccessor::GetKeyValues() const +{ + InternalValueList result; + auto keys = GetKeys(); + result.reserve(keys.size()); + for (auto& key : keys) + { + result.emplace_back(key); + } + + return result; +} + +std::optional IMapAccessor::FindItem(const InternalValue& key) const +{ + if (IsStringValue(key)) + { + const auto name = AsString(key); + if (HasValue(name)) + { + return GetItem(name); + } + } + + return std::nullopt; +} + template class Holder, bool canModify, typename Map = InternalValueMap> class InternalValueMapAdapter : public MapAccessorImpl> { @@ -1710,7 +1737,7 @@ class InternalValueMapAdapter : public MapAccessorImpl> for (const auto& [key, value] : m_values.Get()) { - result.push_back(KeyValuePair{ key, LendItem(m_values, value) }); + result.push_back(KeyValuePair{ InternalValue(key), LendItem(m_values, value) }); } return result; @@ -2044,7 +2071,7 @@ struct OutputValueConvertor result_t operator()(const MapRef& adapter) const { return result_t(adapter.CreateGenericMap()); } result_t operator()(const ListRef& adapter) const { return result_t(adapter.CreateGenericList()); } result_t operator()(const ValueRef& ref) const { return ref.get(); } - result_t operator()(const KeyValuePair& pair) const { return ValuesMap{ { "key", Value(pair.key) }, { "value", IntValue2Value(pair.value) } }; } + result_t operator()(const KeyValuePair& pair) const { return ValuesMap{ { "key", IntValue2Value(pair.key) }, { "value", IntValue2Value(pair.value) } }; } result_t operator()(const Callable&) const { return result_t(); } result_t operator()(const UserCallable&) const { return result_t(); } result_t operator()(const std::shared_ptr&) const { return result_t(); } diff --git a/src/internal_value.h b/src/internal_value.h index 7d207ca09..4f9dcceb2 100644 --- a/src/internal_value.h +++ b/src/internal_value.h @@ -536,6 +536,11 @@ struct IMapAccessor : ValueObject // The pairs in GetKeys() order. The default looks every key up again; adapters that // own a map read its entries directly [[nodiscard]] virtual std::vector GetEntries() const; + // The keys as template values, in GetKeys() order. The default turns each string into a value + [[nodiscard]] virtual InternalValueList GetKeyValues() const; + // The value under `key`, if there is one. The default answers a string key through + // HasValue/GetItem and has no other key, as host maps hold only strings + [[nodiscard]] virtual std::optional FindItem(const InternalValue& key) const; // Const as reading is, so that a value can hold its accessor as a const object; the // accessor changes its own mutable state (docs/tasks/0140). By value: overrides store // the key. `set ns.attr = ...` has no use for the result. @@ -748,8 +753,9 @@ class MapRef [[nodiscard]] InternalValue GetValueByName(const std::string& name) const; [[nodiscard]] const void* GetIdentity() const { return m_accessor ? m_accessor->GetIdentity() : nullptr; } [[nodiscard]] bool HasAttributes() const { return m_accessor != nullptr && m_accessor->HasAttributes(); } - [[nodiscard]] std::vector GetKeys() const { return m_accessor ? m_accessor->GetKeys() : std::vector(); } [[nodiscard]] std::vector GetEntries() const; + [[nodiscard]] InternalValueList GetKeyValues() const; + [[nodiscard]] std::optional FindItem(const InternalValue& key) const; [[nodiscard]] InternalDict* GetMutableItems() const { return m_accessor ? m_accessor->GetMutableItems() : nullptr; } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const { return m_accessor ? m_accessor->GetAttrPolicy() : MapAttrPolicy::KeysOnly; } [[nodiscard]] bool IsNamespace() const { return m_accessor != nullptr && m_accessor->IsNamespace(); } @@ -797,8 +803,9 @@ class MapAdapter [[nodiscard]] InternalValue GetValueByName(const std::string& name) const; [[nodiscard]] const void* GetIdentity() const { return Ref().GetIdentity(); } [[nodiscard]] bool HasAttributes() const { return Ref().HasAttributes(); } - [[nodiscard]] std::vector GetKeys() const { return Ref().GetKeys(); } [[nodiscard]] std::vector GetEntries() const; + [[nodiscard]] InternalValueList GetKeyValues() const; + [[nodiscard]] std::optional FindItem(const InternalValue& key) const; [[nodiscard]] InternalDict* GetMutableItems() const { return Ref().GetMutableItems(); } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const { return Ref().GetAttrPolicy(); } [[nodiscard]] bool IsNamespace() const { return Ref().IsNamespace(); } @@ -1459,7 +1466,7 @@ inline MapAdapter MapRef::ToAdapter() const struct KeyValuePair { - std::string key; + InternalValue key; InternalValue value; }; @@ -1483,6 +1490,26 @@ inline std::vector MapAdapter::GetEntries() const return Ref().GetEntries(); } +inline InternalValueList MapRef::GetKeyValues() const +{ + return m_accessor ? m_accessor->GetKeyValues() : InternalValueList(); +} + +inline InternalValueList MapAdapter::GetKeyValues() const +{ + return Ref().GetKeyValues(); +} + +inline std::optional MapRef::FindItem(const InternalValue& key) const +{ + return m_accessor ? m_accessor->FindItem(key) : std::nullopt; +} + +inline std::optional MapAdapter::FindItem(const InternalValue& key) const +{ + return Ref().FindItem(key); +} + // The list or map a value holds, if it holds one: `if (auto list = AsList(val)) list->GetSize()`. // Not for temporaries, which the view would outlive inline std::optional AsList(const InternalValue& val) diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index 71bc148a0..f7bbcf302 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -27,6 +27,7 @@ #include #include #include +#include using namespace std::string_literals; @@ -41,6 +42,26 @@ struct PrettyPrinter : visitors::BaseVisitor { } + // The entries in the order pprint prints them: by key. String keys sort by code point, + // which is UTF-8 byte order; the keys of a mapping are all strings so far + static void SortEntriesByKey(std::vector& entries) + { + std::vector> order; + order.reserve(entries.size()); + for (size_t idx = 0; idx != entries.size(); ++idx) + { + order.emplace_back(AsString(entries[idx].key), idx); + } + std::sort(order.begin(), order.end()); + std::vector sorted; + sorted.reserve(entries.size()); + for (const auto& item : order) + { + sorted.push_back(std::move(entries[item.second])); + } + entries = std::move(sorted); + } + // A value as pprint prints it: Markup as Markup('...') static std::string Print(const InternalValue& val, const RenderContext* context) { @@ -87,11 +108,11 @@ struct PrettyPrinter : visitors::BaseVisitor fmt::format_to(os, "{{"); // Python's pprint sorts dict keys; UTF-8 byte order is code point order - auto keys = map.GetKeys(); - std::sort(keys.begin(), keys.end()); + auto entries = map.GetEntries(); + SortEntriesByKey(entries); bool isFirst = true; - for (auto& k : keys) + for (auto& entry : entries) { if (isFirst) { @@ -102,8 +123,8 @@ struct PrettyPrinter : visitors::BaseVisitor fmt::format_to(os, ", "); } - fmt::format_to(os, "{}: ", Repr(k)); - fmt::format_to(os, "{}", Print(map.GetValueByName(k), m_context)); + fmt::format_to(os, "{}: ", Print(entry.key, m_context)); + fmt::format_to(os, "{}", Print(entry.value, m_context)); } fmt::format_to(os, "}}"); @@ -117,7 +138,7 @@ struct PrettyPrinter : visitors::BaseVisitor std::string str; auto os = std::back_inserter(str); - fmt::format_to(os, "({}, ", Repr(kwPair.key)); + fmt::format_to(os, "({}, ", Print(kwPair.key, m_context)); fmt::format_to(os, "{})", Print(kwPair.value, m_context)); return str; @@ -433,7 +454,7 @@ class PythonJsonWriter } else if (const auto* pair = GetIf(&value)) { - InternalValueList items{ InternalValue(pair->key), pair->value }; + InternalValueList items{ pair->key, pair->value }; WriteContainer('[', ']', items, level, [this, level](const InternalValue& item) { WriteValue(item, level + 1); }); } else if (auto list = AsList(value)) @@ -448,12 +469,18 @@ class PythonJsonWriter throw std::runtime_error("Object of type "s + objectClass + " is not JSON serializable"); } // sort_keys: Python orders str keys by code point, which is UTF-8 byte order - auto keys = map->GetKeys(); - std::sort(keys.begin(), keys.end()); - WriteContainer('{', '}', keys, level, [this, map, level](const std::string& key) { - WriteString(key); + auto entries = map->GetEntries(); + std::vector> order; + order.reserve(entries.size()); + for (size_t idx = 0; idx != entries.size(); ++idx) + { + order.emplace_back(AsString(entries[idx].key), idx); + } + std::sort(order.begin(), order.end()); + WriteContainer('{', '}', order, level, [this, &entries, level](const std::pair& item) { + WriteString(item.first); m_out += ": "; - WriteValue(map->GetValueByName(key), level + 1); + WriteValue(entries[item.second].value, level + 1); }); } else @@ -594,13 +621,13 @@ InternalValue XmlAttrFilter::Filter(const InternalValue& baseVal, RenderContext& // Port of Jinja2's do_xmlattr: key="escaped str(value)" in the mapping's order, None and // undefined values left out (and callables, which have no str() here) std::string result; - for (auto& key : map->GetKeys()) + for (auto& [keyValue, value] : map->GetEntries()) { - auto value = map->GetValueByName(key); if (IsEmpty(value) || GetIf(&value)) { continue; } + const auto key = AsString(keyValue); // Jinja2 rejects keys with whitespace, "/", ">" or "=" if (std::any_of(key.begin(), key.end(), [](char ch) { return std::strchr(" \t\n\r\f\v/>=", ch) != nullptr && ch != 0; })) { diff --git a/src/statements.cpp b/src/statements.cpp index 29e941ed7..6c3bf1a4f 100644 --- a/src/statements.cpp +++ b/src/statements.cpp @@ -166,7 +166,7 @@ void AssignTo(TargetNodes target, InternalValue value, Sink& sink, RenderContext InternalValueList items; if (auto* pair = GetIf(&value)) { - items.emplace_back(TargetString(pair->key)); + items.push_back(pair->key); items.push_back(pair->value); } else @@ -425,7 +425,7 @@ void AssignLoopTarget(TargetNodes target, const InternalValue& item, ScopeRef sc { slots.items = { &scope.ForName(NameOf(target[1], values.Nodes())), &scope.ForName(NameOf(target[2], values.Nodes())) }; } - *slots.items[0] = TargetString(pair->key); + *slots.items[0] = pair->key; *slots.items[1] = pair->value; return; } @@ -1194,7 +1194,7 @@ void ForStatement::RenderLoopInSlots(const InternalValue& loopVal, OutStream& os else if (const auto* pair = isPair ? GetIf(&curValue) : nullptr) { // `for k, v in d|dictsort` (or d.items()): no list of items to unpack through - values.BindSlot(slots[target[1].slot.value], target[1].hash, TargetString(pair->key)); + values.BindSlot(slots[target[1].slot.value], target[1].hash, pair->key); values.BindSlot(slots[target[2].slot.value], target[2].hash, pair->value); } else diff --git a/src/string_converter_filter.cpp b/src/string_converter_filter.cpp index 97b38e3d1..be6a4f95f 100644 --- a/src/string_converter_filter.cpp +++ b/src/string_converter_filter.cpp @@ -1562,9 +1562,9 @@ InternalValue StringConverter::ApplyUrlEncode(const InternalValue& baseVal, Rend }; if (auto map = AsMap(baseVal)) { - for (auto& key : map->GetKeys()) + for (auto& [key, value] : map->GetEntries()) { - appendPair(InternalValue(key), map->GetValueByName(key)); + appendPair(key, value); } } else if (auto list = AsList(baseVal)) diff --git a/src/value_methods.cpp b/src/value_methods.cpp index 78c098c55..ee80e5ac5 100644 --- a/src/value_methods.cpp +++ b/src/value_methods.cpp @@ -1875,31 +1875,10 @@ InternalDict& MutableDict(const InternalValue& self) return *items; } -// The keys in iteration order: insertion order for the dicts the template owns -std::vector KeysOf(const MapRef& map) -{ - if (auto* items = map.GetMutableItems()) - { - std::vector keys; - keys.reserve(items->size()); - for (auto& [key, value] : *items) - { - keys.push_back(key); - } - return keys; - } - return map.GetKeys(); -} - InternalValue DictKeys(const InternalValue& self, const CallParams& params, RenderContext&) { CheckArgs(params, "keys", 0, 0); - InternalValueList result; - for (auto& key : KeysOf(MapOf(self))) - { - result.emplace_back(key); - } - return ListAdapter::CreateAdapter(std::move(result)); + return ListAdapter::CreateAdapter(MapOf(self).GetKeyValues()); } InternalValue DictValues(const InternalValue& self, const CallParams& params, RenderContext&) @@ -1907,9 +1886,9 @@ InternalValue DictValues(const InternalValue& self, const CallParams& params, Re CheckArgs(params, "values", 0, 0); const auto& map = MapOf(self); InternalValueList result; - for (auto& key : KeysOf(map)) + for (auto& entry : map.GetEntries()) { - result.push_back(map.GetValueByName(key)); + result.push_back(std::move(entry.value)); } InternalValue list = ListAdapter::CreateAdapter(std::move(result)); if (self.ShouldExtendLifetime()) @@ -1924,9 +1903,9 @@ InternalValue DictItems(const InternalValue& self, const CallParams& params, Ren CheckArgs(params, "items", 0, 0); const auto& map = MapOf(self); InternalValueList result; - for (auto& key : KeysOf(map)) + for (auto& entry : map.GetEntries()) { - result.push_back(MakeTuple({ InternalValue(key), map.GetValueByName(key) })); + result.push_back(MakeTuple({ std::move(entry.key), std::move(entry.value) })); } InternalValue list = ListAdapter::CreateAdapter(std::move(result)); if (self.ShouldExtendLifetime()) @@ -1994,9 +1973,9 @@ void CollectPositionalUpdates(const InternalValue& other, DictUpdates& updates) // An object such as namespace() is no mapping, and not iterable either if (auto otherMap = AsMap(other); otherMap && !otherMap->GetObjectClass()) { - for (auto& key : KeysOf(*otherMap)) + for (auto& entry : otherMap->GetEntries()) { - updates.emplace_back(key, otherMap->GetValueByName(key)); + updates.emplace_back(KeyString(entry.key), std::move(entry.value)); } } else @@ -2071,9 +2050,9 @@ InternalValue DictCopy(const InternalValue& self, const CallParams& params, Rend CheckArgs(params, "copy", 0, 0); const auto& map = MapOf(self); InternalDict result; - for (auto& key : KeysOf(map)) + for (auto& entry : map.GetEntries()) { - result[key] = map.GetValueByName(key); + result[KeyString(entry.key)] = std::move(entry.value); } return CreateMapAdapter(std::move(result)); } @@ -2327,14 +2306,13 @@ InternalValue MakeMutable(const InternalValue& value) if (auto map = AsMap(value)) { InternalDict items; - for (auto& key : map->GetKeys()) + for (auto& [key, item] : map->GetEntries()) { - auto item = map->GetValueByName(key); if (extendLifetime) { item.SetParentData(value); } - items[key] = std::move(item); + items[KeyString(key)] = std::move(item); } return CreateMapAdapter(std::move(items)); } diff --git a/src/value_visitors.h b/src/value_visitors.h index 5a66a30bf..2284accdc 100644 --- a/src/value_visitors.h +++ b/src/value_visitors.h @@ -825,21 +825,20 @@ void ValueRendererBase::operator()(const MapRef& map) const // Python prints dicts in their own order: ValuesMap and InternalDict keep insertion order, reflected // structs report sorted keys - auto keys = map.GetKeys(); + auto entries = map.GetEntries(); - ValueRenderer keyRenderer(*m_os, true); AppendAscii("{"); bool isFirst = true; - for (auto& key : keys) + for (auto& [key, value] : entries) { if (!isFirst) { AppendAscii(", "); } isFirst = false; - keyRenderer(key); + RenderRepr(key, &containers); AppendAscii(": "); - RenderRepr(map.GetValueByName(key), &containers); + RenderRepr(value, &containers); } AppendAscii("}"); if (objectClass) @@ -854,7 +853,7 @@ void ValueRendererBase::operator()(const KeyValuePair& pair) const { // dict items are (key, value) tuples in Python AppendAscii("("); - ValueRenderer(*m_os, true)(pair.key); + RenderRepr(pair.key, m_containers); AppendAscii(", "); RenderRepr(pair.value, m_containers); AppendAscii(")"); @@ -1730,9 +1729,10 @@ struct BinaryMathOperation : BaseVisitor<> { CheckStack(); equal = true; - for (auto& key : left.GetKeys()) + for (auto& [key, value] : left.GetEntries()) { - if (!right.HasValue(key) || !ConvertToBool(Apply2(left.GetValueByName(key), right.GetValueByName(key), BinaryExpression::LogicalEq))) + const auto other = right.FindItem(key); + if (!other || !ConvertToBool(Apply2(value, *other, BinaryExpression::LogicalEq))) { equal = false; break; From 51c371bb18788a822c73ce87ad79799c5c0992d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 11:44:27 +0000 Subject: [PATCH 02/18] Non-string mapping keys in template dicts (0036) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- CMakeLists.txt | 1 + docs/tasks/0036-non-string-mapping-keys.md | 22 +- src/expression_evaluator.cpp | 32 +- src/filters.cpp | 34 +- src/global_functions.cpp | 29 +- src/internal_value.cpp | 156 ++- src/internal_value.h | 11 +- src/serialize_filters.cpp | 131 +- src/template_dict.cpp | 450 +++++++ src/template_dict.h | 111 ++ src/testers.cpp | 17 +- src/undefined.cpp | 2 +- src/value_methods.cpp | 87 +- test/internal_value_test.cpp | 152 +++ test/parity/cases/errors.py | 6 + test/parity/cases/filters.py | 19 + test/parity/cases/globals.py | 4 + test/parity/cases/literals.py | 7 + test/parity/cases/methods.py | 9 + test/parity/cases/operators.py | 3 + test/parity/cases/statements.py | 2 + test/parity/cases/subscripts.py | 3 + test/parity/divergences/errors.txt | 1 - test/parity/divergences/literals.txt | 1 - test/parity/expected/errors.json | 114 ++ test/parity/expected/filters.json | 1282 ++++++++++++++++++++ test/parity/expected/globals.json | 36 + test/parity/expected/literals.json | 56 + test/parity/expected/methods.json | 212 ++++ test/parity/expected/operators.json | 68 +- test/parity/expected/statements.json | 80 ++ test/parity/expected/subscripts.json | 96 ++ 32 files changed, 3090 insertions(+), 144 deletions(-) create mode 100644 src/template_dict.cpp create mode 100644 src/template_dict.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ccb4dad6a..196c07eda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,6 +172,7 @@ set(JINJA2CPP_SOURCES src/statements.cpp src/string_converter_filter.cpp src/template.cpp + src/template_dict.cpp src/template_env.cpp src/template_parser.cpp src/tester_factories.cpp diff --git a/docs/tasks/0036-non-string-mapping-keys.md b/docs/tasks/0036-non-string-mapping-keys.md index 656447671..7bf3e937d 100644 --- a/docs/tasks/0036-non-string-mapping-keys.md +++ b/docs/tasks/0036-non-string-mapping-keys.md @@ -1,5 +1,5 @@ --- -status: open +status: done priority: low area: parity depends: [0013] @@ -22,3 +22,23 @@ Full fix: a key type in the value model (a variant of string, integer, bool, Non tuple), which touches the public API and is worth doing only together with 0031. **Done when.** No line of `test/parity/divergences/` names task 0036, and `ctest -R parity` passes. + +**Shipped (D13 = b).** Dict literals, `dict()`, `namespace()` and the results of the dict +methods take any hashable key (`src/template_dict.*`): `1`, `1.0` and `True` are one key +(first spelling and position kept, last value wins), so are `(1, 2.0)` and `(1.0, 2)`; +`none`, floats, tuples and ranges are keys; a list or a dict raises `TypeError: unhashable +type`. Subscript, `in`, `get`, `setdefault`, `update`, `pop`, `popitem`, `copy`, iteration, +`items`, `dictsort`, `pprint`, `tojson`, `xmlattr`, `urlencode` and `==` follow Python. The +public `ValuesMap`/`GenericMap` keys stay `std::string`. + +*Conversion rule at the public boundary* (`CreateGenericMap` of a template dict, what a user +callable receives): a dict with a key that is not a string is handed over as a snapshot, not a +live view, keyed by `str(key)` (`{1: 'x', None: 'y'}` has the keys `"1"` and `"None"`). When two +keys spell alike (`{1: 'x', '1': 'y'}`) the later value wins at the position of the first, as +Python's `{str(k): v for k, v in d.items()}` has it. A dict of string keys stays a live view. +Changing this rule later is a break for callers who rely on it. + +*Not done:* object keys (`{namespace(): 1}`, host objects, callables, undefined) have no +identity token yet and all read as the empty string, as before; that is the first bullet of +0212 (`globals.object_dict_key` stays listed). All NaN keys are one key (Python compares NaN by +identity). diff --git a/src/expression_evaluator.cpp b/src/expression_evaluator.cpp index d32d06689..178823733 100644 --- a/src/expression_evaluator.cpp +++ b/src/expression_evaluator.cpp @@ -10,6 +10,7 @@ #include "python_format.h" #include "recursion_guard.h" #include "render_context.h" +#include "template_dict.h" #include "testers.h" #include "undefined.h" #include "value_methods.h" @@ -844,29 +845,6 @@ InternalValue TupleCreator::Evaluate(RenderContext& context) return list; } -namespace -{ -// Mapping keys are strings in the value model; integer and boolean keys are stored as -// their decimal or Python spelling, so {1: 'x'} has the key '1' -struct DictKeyGetter : public visitors::BaseVisitor -{ - using BaseVisitor::operator(); - - template - std::string operator()(const std::basic_string& str) const - { - return ConvertString(str); - } - template - std::string operator()(const std::basic_string_view& str) const - { - return ConvertString(str); - } - std::string operator()(int64_t val) const { return std::to_string(val); } - std::string operator()(bool val) const { return val ? "True" : "False"; } -}; -} // namespace - bool DictCreator::IsFoldable(const NodeArena& nodes) const { const auto items = nodes[m_exprs]; @@ -877,13 +855,15 @@ InternalValue DictCreator::Evaluate(RenderContext& context) { CheckStack(); const auto nodes = context.Nodes(); - InternalDict result; + TemplateDict result; for (const auto& item : nodes[m_exprs]) { // Python evaluates the key before the value; an assignment does not fix that order - auto key = Apply(nodes[item.key].Evaluate(context)); + const auto key = nodes[item.key].Evaluate(context); auto value = nodes[item.value].Evaluate(context); - result[std::move(key)] = std::move(value); + // Any hashable key; an unhashable one is a TypeError. A repeated key (1 and 1.0) keeps + // its first spelling and position and takes the last value + result.Set(key, std::move(value)); } return CreateMapAdapter(std::move(result)); diff --git a/src/filters.cpp b/src/filters.cpp index 6a807373c..601a663ab 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -8,6 +8,7 @@ #include "out_stream.h" // IWYU pragma: keep (GetStreamOnString returns an OutStream by value) #include "render_context.h" #include "string_builder.h" +#include "template_dict.h" #include "testers.h" #include "undefined.h" #include "unicode_tables.h" @@ -986,37 +987,6 @@ struct UniqueItem int64_t idx; }; -// The type Python's hash() rejects in a value, if any: a list or a dict, also inside a -// tuple. A range hashes, and so does an object stored as a map (namespace(), cycler()) or -// a host object (a reflected struct, which Python would hold as an object, or a JSON object) -struct UnhashableTypeGetter : visitors::BaseVisitor -{ - using visitors::BaseVisitor::operator(); - - const char* operator()(const ListRef& list) const - { - if (list.GetRangeInfo()) - { - return nullptr; - } - if (!list.IsTuple()) - { - return "list"; - } - const char* result = nullptr; - list.ForEach([&result](const InternalValue& item) { - result = Apply(item); - return result == nullptr; - }); - return result; - } - const char* operator()(const MapRef& map) const - { - return map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst ? nullptr : "dict"; - } - const char* operator()(const KeyValuePair& pair) const { return Apply(pair.value); } -}; - // Drops repeated keys, keeping the first of each in its original position void DropDuplicates(std::vector& items, BinaryExpression::CompareType compType) { @@ -1077,7 +1047,7 @@ InternalValue Unique(const ListAdapter& list, // Jinja2 keeps a set of the keys it saw for (const auto& item : items) { - if (const char* type = Apply(item.val)) + if (const char* type = UnhashableType(item.val)) { throw std::runtime_error("unhashable type: '"s + type + "'"); } diff --git a/src/global_functions.cpp b/src/global_functions.cpp index 58d5a9c27..d5a7eaa6b 100644 --- a/src/global_functions.cpp +++ b/src/global_functions.cpp @@ -2,6 +2,7 @@ #include "internal_value.h" #include "markup.h" #include "python_format.h" +#include "template_dict.h" #include "undefined.h" #include "value_visitors.h" @@ -78,19 +79,6 @@ ParsedArguments ParseArgs(const std::initializer_list& argsInfo, c return args; } -std::string KeyToString(const InternalValue& key) -{ - if (IsStringValue(key)) - { - return AsString(key); - } - - // Mapping keys are strings (task 0036): store other keys by their printed form - std::string result; - Apply>(key, result); - return result; -} - // Python's range and randrange take integers (bool among them) and nothing else; a plain // undefined is a missing argument, left to the caller void CheckIntegerArgument(const InternalValue& arg) @@ -140,8 +128,7 @@ InternalValue CallRange(const CallParams& params, RenderContext&) } // The items of dict(mapping_or_pairs, **kwargs), into `result` -template -void CollectDictItems(const CallParams& params, const char* fnName, Map& result) +void CollectDictItems(const CallParams& params, const char* fnName, TemplateDict& result) { if (params.posParams.size() > 1) { @@ -161,7 +148,7 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) { for (auto& [key, value] : map->GetEntries()) { - result[KeyToString(key)] = std::move(value); + result.Set(key, std::move(value)); } } else if (auto list = AsList(source)) @@ -170,7 +157,7 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) { if (const auto* pair = GetIf(&item)) { - result[KeyToString(pair->key)] = pair->value; + result.Set(pair->key, pair->value); continue; } // Any iterable of two items is a pair, a two-character string included @@ -185,7 +172,7 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) { throw std::runtime_error("dictionary update sequence element has wrong length; 2 is required"); } - result[KeyToString(pair[0])] = pair[1]; + result.Set(pair[0], pair[1]); } } else if (!IsEmpty(source)) @@ -196,14 +183,14 @@ void CollectDictItems(const CallParams& params, const char* fnName, Map& result) for (const auto& [name, value] : params.kwParams) { - result[name] = value; + result.SetString(name, value); } } // dict(mapping_or_pairs, **kwargs) InternalValue CallDict(const CallParams& params, RenderContext&) { - InternalDict result; + TemplateDict result; CollectDictItems(params, "dict", result); return CreateMapAdapter(std::move(result)); } @@ -211,7 +198,7 @@ InternalValue CallDict(const CallParams& params, RenderContext&) // namespace(mapping_or_pairs, **kwargs): takes its arguments as dict() does InternalValue CallNamespace(const CallParams& params, RenderContext&) { - InternalDict result; + TemplateDict result; CollectDictItems(params, "namespace", result); return CreateNamespaceAdapter(std::move(result)); } diff --git a/src/internal_value.cpp b/src/internal_value.cpp index 47daf8296..8d72cf6d3 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -2,6 +2,7 @@ #include "expression_evaluator.h" #include "generic_adapters.h" +#include "template_dict.h" #include "undefined.h" #include "value_visitors.h" @@ -434,7 +435,28 @@ struct SubscriptionVisitor : public visitors::BaseVisitor<> return values.GetValueByIndex(index); } - InternalValue operator()(const MapRef& /*values*/, int64_t /*index*/) const { return InternalValue(); } + // A key that is not a string is a key of a dict the template owns (1, 1.5, True, none, a tuple); + // other maps hold strings only, and an unhashable key reads as undefined, as Jinja2 swallows + // the TypeError + [[nodiscard]] static InternalValue FindKey(const MapRef& values, const InternalValue& key) + { + auto found = values.FindItem(key); + return found ? std::move(*found) : InternalValue(); + } + + InternalValue operator()(const MapRef& values, int64_t key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, bool key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, double key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, NoneValue /*key*/) const { return FindKey(values, InternalValue(NoneValue{})); } + InternalValue operator()(const MapRef& values, const KeyValuePair& key) const { return FindKey(values, InternalValue(key)); } + InternalValue operator()(const MapRef& values, const ListRef& key) const + { + if (!key.IsTuple() && !key.GetRangeInfo()) + { + return InternalValue(); + } + return FindKey(values, InternalValue(key.ToAdapter())); + } // A dictsort pair reads as the 2-tuple (key, value) InternalValue operator()(const KeyValuePair& pair, int64_t index) const @@ -1771,19 +1793,122 @@ class InternalValueMapAdapter : public MapAccessorImpl m_values; }; -// A dict the template owns (dict literals, kwargs, dict()): shared like a Python dict -class SharedDictAdapter : public InternalValueMapAdapter +// A dict the template owns (dict literals, kwargs, dict()): shared like a Python dict, and keyed +// like one (TemplateDict). A string key costs one lookup as in any other map +class SharedDictAdapter : public MapAccessorImpl { public: - using InternalValueMapAdapter::InternalValueMapAdapter; + explicit SharedDictAdapter(TemplateDict&& values) + : m_values(std::move(values)) + { + } + + [[nodiscard]] size_t GetSize() const override { return m_values.Get().size(); } + [[nodiscard]] bool HasValue(const std::string& name) const override { return m_values.Get().FindString(name) != m_values.Get().end(); } + [[nodiscard]] InternalValue GetItem(const std::string& name) const override + { + auto& vals = m_values.Get(); + auto p = vals.FindString(name); + return p == vals.end() ? InternalValue() : p->second; + } + // The keys spelled as str(), so that a reader of the string keyed view never sees a token + [[nodiscard]] std::vector GetKeys() const override + { + std::vector result; + auto& vals = m_values.Get(); + result.reserve(vals.size()); + for (const auto& [stored, value] : vals) + { + result.push_back(DictKeyText(vals.KeyOf(stored))); + } + return result; + } + [[nodiscard]] InternalValueList GetKeyValues() const override + { + InternalValueList result; + auto& vals = m_values.Get(); + result.reserve(vals.size()); + for (const auto& [stored, value] : vals) + { + result.push_back(vals.KeyOf(stored)); + } + + return result; + } + [[nodiscard]] std::vector GetEntries() const override + { + std::vector result; + auto& vals = m_values.Get(); + result.reserve(vals.size()); + for (const auto& [stored, value] : vals) + { + result.push_back(KeyValuePair{ vals.KeyOf(stored), value }); + } + + return result; + } + [[nodiscard]] std::optional FindItem(const InternalValue& key) const override + { + auto& vals = m_values.Get(); + auto p = vals.Find(key); + if (p == vals.end()) + { + return std::nullopt; + } + return p->second; + } + + bool SetValue(std::string name, const InternalValue& val) const override + { + m_values.Get().SetString(std::move(name), val); + return true; + } + [[nodiscard]] bool ShouldExtendLifetime() const override { return m_values.ShouldExtendLifetime(); } + [[nodiscard]] bool IsEqual(const IComparable& other) const override + { + auto* val = dynamic_cast(&other); + if (!val) + { + return false; + } + return m_values == val->m_values; + } [[nodiscard]] const void* GetIdentity() const override { return &m_values.Get(); } - [[nodiscard]] InternalDict* GetMutableItems() const override { return &m_values.Get(); } + [[nodiscard]] TemplateDict* GetMutableItems() const override { return &m_values.Get(); } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const override { return MapAttrPolicy::MethodsFirst; } [[nodiscard]] GenericMap CreateGenericMap() const override { + if (auto snapshot = CreateSnapshot()) + { + return std::move(*snapshot); + } return GenericMap([accessor = *this]() -> const IMapItemAccessor* { return &accessor; }); } + +protected: + // The public map has string keys only (D13: docs/release-2.0.md, docs/tasks/0036). A dict + // with a key that is not a string is handed over as a copy, not a live view, with the keys + // spelled as str() does: {1: 'x', None: 'y'} has the keys "1" and "None". Keys that spell + // alike ({1: 'x', '1': 'y'}) collapse, and the later value wins at the position of the first, + // as Python's {str(k): v for k, v in d.items()} has it. A dict of string keys stays a live view + [[nodiscard]] std::optional CreateSnapshot() const + { + auto& vals = m_values.Get(); + if (!vals.HasKeyValues()) + { + return std::nullopt; + } + ValuesMap snapshot; + for (const auto& [stored, value] : vals) + { + snapshot[DictKeyText(vals.KeyOf(stored))] = IntValue2Value(value); + } + return CreateMapAdapter(std::move(snapshot)).CreateGenericMap(); + } + + // Mutable for SetValue, the one writer (IMapAccessor::SetValue) + mutable BySharedMutable m_values; }; // namespace(): shared like a dict, but an object with attributes rather than a dict to the @@ -1793,11 +1918,20 @@ class NamespaceAdapter : public SharedDictAdapter public: using SharedDictAdapter::SharedDictAdapter; + // Its attributes are names: ns[1] is undefined, as in Python + [[nodiscard]] std::optional FindItem(const InternalValue& key) const override + { + return IsStringValue(key) ? SharedDictAdapter::FindItem(key) : std::nullopt; + } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const override { return MapAttrPolicy::KeysOnly; } [[nodiscard]] bool IsNamespace() const override { return true; } [[nodiscard]] const char* GetObjectClass() const override { return "Namespace"; } [[nodiscard]] GenericMap CreateGenericMap() const override { + if (auto snapshot = CreateSnapshot()) + { + return std::move(*snapshot); + } return GenericMap([accessor = *this]() -> const IMapItemAccessor* { return &accessor; }); } }; @@ -2013,15 +2147,25 @@ MapAdapter CreateMapAdapter(InternalValueMap&& values) } MapAdapter CreateMapAdapter(InternalDict&& values) +{ + return MapAdapter(MakeRef(TemplateDict(std::move(values)))); +} + +MapAdapter CreateMapAdapter(TemplateDict&& values) { return MapAdapter(MakeRef(std::move(values))); } -MapAdapter CreateNamespaceAdapter(InternalDict&& values) +MapAdapter CreateNamespaceAdapter(TemplateDict&& values) { return MapAdapter(MakeRef(std::move(values))); } +MapAdapter CreateNamespaceAdapter(InternalDict&& values) +{ + return MapAdapter(MakeRef(TemplateDict(std::move(values)))); +} + MapAdapter CreateObjectAdapter(InternalValueMap&& values, const char* className) { return MapAdapter(MakeRef(std::move(values), className)); diff --git a/src/internal_value.h b/src/internal_value.h index 4f9dcceb2..2c22e8119 100644 --- a/src/internal_value.h +++ b/src/internal_value.h @@ -413,6 +413,9 @@ void ReserveHint(List& list, size_t size) // dicts do; scopes and other lookup-only maps stay InternalValueMap (docs/tasks/0031) using InternalDict = OrderedMap; +// A dict with Python's keys: any hashable value, over the string keys of InternalDict (template_dict.h) +class TemplateDict; + struct IListAccessor; struct IListAccessorEnumerator; @@ -554,7 +557,7 @@ struct IMapAccessor : ValueObject // See IListAccessor::GetIdentity [[nodiscard]] virtual const void* GetIdentity() const { return this; } // See IListAccessor::GetMutableItems - [[nodiscard]] virtual InternalDict* GetMutableItems() const { return nullptr; } + [[nodiscard]] virtual TemplateDict* GetMutableItems() const { return nullptr; } // How x.name resolves against Python's dict methods (items, get, ...) [[nodiscard]] virtual MapAttrPolicy GetAttrPolicy() const { return MapAttrPolicy::KeysOnly; } // A namespace() object, the only one `set obj.attr = ...` can change @@ -756,7 +759,7 @@ class MapRef [[nodiscard]] std::vector GetEntries() const; [[nodiscard]] InternalValueList GetKeyValues() const; [[nodiscard]] std::optional FindItem(const InternalValue& key) const; - [[nodiscard]] InternalDict* GetMutableItems() const { return m_accessor ? m_accessor->GetMutableItems() : nullptr; } + [[nodiscard]] TemplateDict* GetMutableItems() const { return m_accessor ? m_accessor->GetMutableItems() : nullptr; } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const { return m_accessor ? m_accessor->GetAttrPolicy() : MapAttrPolicy::KeysOnly; } [[nodiscard]] bool IsNamespace() const { return m_accessor != nullptr && m_accessor->IsNamespace(); } [[nodiscard]] const char* GetObjectClass() const { return m_accessor ? m_accessor->GetObjectClass() : nullptr; } @@ -806,7 +809,7 @@ class MapAdapter [[nodiscard]] std::vector GetEntries() const; [[nodiscard]] InternalValueList GetKeyValues() const; [[nodiscard]] std::optional FindItem(const InternalValue& key) const; - [[nodiscard]] InternalDict* GetMutableItems() const { return Ref().GetMutableItems(); } + [[nodiscard]] TemplateDict* GetMutableItems() const { return Ref().GetMutableItems(); } [[nodiscard]] MapAttrPolicy GetAttrPolicy() const { return Ref().GetAttrPolicy(); } [[nodiscard]] bool IsNamespace() const { return Ref().IsNamespace(); } [[nodiscard]] const char* GetObjectClass() const { return Ref().GetObjectClass(); } @@ -1353,8 +1356,10 @@ using InternalValueMap = robin_hood::unordered_map { } + // pprint's _safe_key: `a < b` when Python can order the two keys, else by the name of + // their types (str(type)), so that {1: 'a', 'b': 2} prints with the int first. Two keys + // of one type that cannot be ordered (None and None) stay where they are, as Python + // compares their ids + static bool SafeKeyLess(const InternalValue& left, const InternalValue& right) + { + const auto isNumber = [](const InternalValue& val) { + return val.Kind() == ValueKind::Int || val.Kind() == ValueKind::Double || val.Kind() == ValueKind::Bool; + }; + const auto isSequence = [](const InternalValue& val) { return val.Kind() == ValueKind::List || val.Kind() == ValueKind::KeyValuePair; }; + if ((isNumber(left) && isNumber(right)) || (IsStringValue(left) && IsStringValue(right)) || (isSequence(left) && isSequence(right))) + { + try + { + return ConvertToBool(Apply2(left, right, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive)); + } + catch (const std::runtime_error&) + { + // Tuples with items Python cannot order: ordered by type like the rest + } + } + return std::strcmp(Apply(left), Apply(right)) < 0; + } + // The entries in the order pprint prints them: by key. String keys sort by code point, - // which is UTF-8 byte order; the keys of a mapping are all strings so far + // which is UTF-8 byte order static void SortEntriesByKey(std::vector& entries) { + if (!std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); })) + { + std::stable_sort(entries.begin(), entries.end(), [](const KeyValuePair& left, const KeyValuePair& right) { return SafeKeyLess(left.key, right.key); }); + return; + } std::vector> order; order.reserve(entries.size()); for (size_t idx = 0; idx != entries.size(); ++idx) @@ -435,18 +464,7 @@ class PythonJsonWriter } else if (const auto* d = GetIf(&value)) { - if (std::isnan(*d)) - { - m_out += "NaN"; - } - else if (std::isinf(*d)) - { - m_out += *d < 0 ? "-Infinity" : "Infinity"; - } - else - { - m_out += visitors::PythonFloatText(*d).View(); - } + m_out += JsonFloatText(*d); } else if (auto str = GetAsSameString(std::string(), value)) { @@ -472,11 +490,18 @@ class PythonJsonWriter auto entries = map->GetEntries(); std::vector> order; order.reserve(entries.size()); - for (size_t idx = 0; idx != entries.size(); ++idx) + if (std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); })) + { + for (size_t idx = 0; idx != entries.size(); ++idx) + { + order.emplace_back(AsString(entries[idx].key), idx); + } + std::sort(order.begin(), order.end()); + } + else { - order.emplace_back(AsString(entries[idx].key), idx); + OrderNonStringKeys(entries, order); } - std::sort(order.begin(), order.end()); WriteContainer('{', '}', order, level, [this, &entries, level](const std::pair& item) { WriteString(item.first); m_out += ": "; @@ -489,6 +514,75 @@ class PythonJsonWriter } } + // How json.dumps writes a float, also as a key + static std::string JsonFloatText(double val) + { + if (std::isnan(val)) + { + return "NaN"; + } + if (std::isinf(val)) + { + return val < 0 ? "-Infinity" : "Infinity"; + } + return std::string(visitors::PythonFloatText(val).View()); + } + + // Python's `a < b` for two keys that json.dumps accepts: numbers (bool included) order + // among themselves, strings among themselves, and anything else is a TypeError + static bool KeyLess(const InternalValue& left, const InternalValue& right) + { + const bool leftString = IsStringValue(left); + const bool rightString = IsStringValue(right); + if (leftString && rightString) + { + return AsString(left) < AsString(right); + } + if (leftString || rightString || left.IsNone() || right.IsNone()) + { + throw std::runtime_error("'<' not supported between instances of '"s + Apply(left) + "' and '" + Apply(right) + "'"); + } + return ConvertToBool(Apply2(left, right, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive)); + } + + // The keys of a mapping with a key that is not a string, in sort_keys order, with the JSON + // spelling of each: 1 is "1", 2.5 is "2.5", True is "true" and None is "null". A tuple or + // an object is no JSON key; keys of types Python cannot compare (a str and an int) are a TypeError + static void OrderNonStringKeys(const std::vector& entries, std::vector>& order) + { + for (size_t idx = 0; idx != entries.size(); ++idx) + { + const auto& key = entries[idx].key; + std::string text; + if (const auto* i = GetIf(&key)) + { + text = std::to_string(*i); + } + else if (const auto* b = GetIf(&key)) + { + text = *b ? "true" : "false"; + } + else if (const auto* d = GetIf(&key)) + { + text = JsonFloatText(*d); + } + else if (key.IsNone()) + { + text = "null"; + } + else if (IsStringValue(key)) + { + text = AsString(key); + } + else + { + throw std::runtime_error("keys must be str, int, float, bool or None, not "s + Apply(key)); + } + order.emplace_back(std::move(text), idx); + } + std::stable_sort(order.begin(), order.end(), [&entries](const auto& left, const auto& right) { return KeyLess(entries[left.second].key, entries[right.second].key); }); + } + RenderContext& m_context; std::optional m_indent; StringBuilder m_out; @@ -627,6 +721,11 @@ InternalValue XmlAttrFilter::Filter(const InternalValue& baseVal, RenderContext& { continue; } + // Jinja2 searches the key for whitespace with a regular expression, which only takes text + if (!IsStringValue(keyValue)) + { + throw std::runtime_error("expected string or bytes-like object, got '"s + Apply(keyValue) + "'"); + } const auto key = AsString(keyValue); // Jinja2 rejects keys with whitespace, "/", ">" or "=" if (std::any_of(key.begin(), key.end(), [](char ch) { return std::strchr(" \t\n\r\f\v/>=", ch) != nullptr && ch != 0; })) diff --git a/src/template_dict.cpp b/src/template_dict.cpp new file mode 100644 index 000000000..8c118334b --- /dev/null +++ b/src/template_dict.cpp @@ -0,0 +1,450 @@ +#include "template_dict.h" + +#include "value_visitors.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace jinja2 +{ +namespace +{ +// The kind byte after the mark. Built from char constants, never from string literals: a +// token holds NUL bytes, and "\xFFf" would read as the single character \xFFF +constexpr char KindInt = 'n'; +constexpr char KindFloat = 'g'; +constexpr char KindNone = 'N'; +constexpr char KindTuple = 't'; +constexpr char KindRange = 'r'; +constexpr char KindString = 's'; + +// The type of a value Python's hash() rejects, also inside a tuple +struct UnhashableTypeGetter : visitors::BaseVisitor +{ + using visitors::BaseVisitor::operator(); + + const char* operator()(const ListRef& list) const + { + if (list.GetRangeInfo()) + { + return nullptr; + } + if (!list.IsTuple()) + { + return "list"; + } + const char* result = nullptr; + list.ForEach([&result](const InternalValue& item) { + result = Apply(item); + return result == nullptr; + }); + return result; + } + const char* operator()(const MapRef& map) const + { + return map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst ? nullptr : "dict"; + } + const char* operator()(const KeyValuePair& pair) const { return Apply(pair.value); } +}; + +template +void AppendBytes(std::string& out, T value) +{ + char bytes[sizeof(T)]; + std::memcpy(bytes, &value, sizeof(T)); + out.append(bytes, sizeof(T)); +} + +void AppendInt(std::string& out, int64_t value) +{ + out.push_back(DictKeyMark); + out.push_back(KindInt); + AppendBytes(out, value); +} + +// The number of items of range(start, stop, step), for a step that is not zero +uint64_t RangeLength(const RangeInfo& range) +{ + if (range.step > 0 ? range.stop <= range.start : range.stop >= range.start) + { + return 0; + } + // Unsigned, so that the distance of two int64 values cannot overflow + if (range.step > 0) + { + const auto distance = static_cast(range.stop) - static_cast(range.start); + return ((distance - 1) / static_cast(range.step)) + 1; + } + const auto distance = static_cast(range.start) - static_cast(range.stop); + const auto stride = uint64_t{ 0 } - static_cast(range.step); + return ((distance - 1) / stride) + 1; +} + +// Appends the token of a value, and returns the type that made it unhashable instead +struct KeyTokenBuilder +{ + explicit KeyTokenBuilder(std::string& out) + : m_out(out) + { + } + + template + const char* operator()(const std::basic_string& str) const + { + AddString(std::basic_string_view(str)); + return nullptr; + } + template + const char* operator()(const std::basic_string_view& str) const + { + AddString(str); + return nullptr; + } + const char* operator()(bool val) const + { + AppendInt(m_out, val ? 1 : 0); + return nullptr; + } + const char* operator()(int64_t val) const + { + AppendInt(m_out, val); + return nullptr; + } + const char* operator()(double val) const + { + // 2^63: the first double that is no int64 + constexpr double limit = 9223372036854775808.0; + if (val == std::floor(val) && val >= -limit && val < limit) + { + // 1.0 is the key 1, and -0.0 the key 0 + AppendInt(m_out, static_cast(val)); + return nullptr; + } + // Python compares NaN by identity; one NaN key here, a deliberate divergence + const auto number = std::isnan(val) ? std::numeric_limits::quiet_NaN() : val; + m_out.push_back(DictKeyMark); + m_out.push_back(KindFloat); + AppendBytes(m_out, number); + return nullptr; + } + const char* operator()(NoneValue /*val*/) const + { + m_out.push_back(DictKeyMark); + m_out.push_back(KindNone); + return nullptr; + } + const char* operator()(const ListRef& list) const + { + if (const auto* range = list.GetRangeInfo()) + { + // Equal ranges are equal keys: an empty range is one, and a single item has no step + auto length = RangeLength(*range); + auto start = range->start; + auto step = range->step; + if (length == 0) + { + start = 0; + step = 0; + } + else if (length == 1) + { + step = 1; + } + m_out.push_back(DictKeyMark); + m_out.push_back(KindRange); + AppendBytes(m_out, length); + AppendBytes(m_out, start); + AppendBytes(m_out, step); + return nullptr; + } + if (!list.IsTuple()) + { + return "list"; + } + m_out.push_back(DictKeyMark); + m_out.push_back(KindTuple); + AppendBytes(m_out, static_cast(list.GetSize().value_or(0))); + const char* result = nullptr; + list.ForEach([this, &result](const InternalValue& item) { + result = AddElement(item); + return result == nullptr; + }); + return result; + } + const char* operator()(const MapRef& map) const + { + if (map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst) + { + AddObject(); + return nullptr; + } + return "dict"; + } + // A dictsort pair is the tuple (key, value) + const char* operator()(const KeyValuePair& pair) const + { + m_out.push_back(DictKeyMark); + m_out.push_back(KindTuple); + AppendBytes(m_out, uint32_t{ 2 }); + if (const char* type = AddElement(pair.key)) + { + return type; + } + return AddElement(pair.value); + } + // Undefined values, callables and renderers + template + const char* operator()(const T& /*val*/) const + { + AddObject(); + return nullptr; + } + +private: + // A tuple item: the length of its token, then the token + const char* AddElement(const InternalValue& item) const + { + const auto lengthAt = m_out.size(); + AppendBytes(m_out, uint32_t{ 0 }); + const auto start = m_out.size(); + if (const char* type = Apply(item, m_out)) + { + return type; + } + const auto length = static_cast(m_out.size() - start); + std::memcpy(&m_out[lengthAt], &length, sizeof(length)); + return nullptr; + } + + // Objects, undefined values and callables have no key of their own yet (docs/tasks/0212): + // they add nothing, so as a key each stands for the empty string, as dict keys did before + static void AddObject() {} + + template + void AddString(std::basic_string_view str) const + { + if constexpr (std::is_same_v) + { + if (!str.empty() && str.front() == DictKeyMark) + { + m_out.push_back(DictKeyMark); + m_out.push_back(KindString); + } + m_out.append(str); + } + else + { + // Keys are narrow UTF-8 whatever the template's character type + const auto text = ConvertString(str); + AddString(std::string_view(text)); + } + } + + std::string& m_out; +}; + +std::string EscapeString(const std::string& name) +{ + std::string token; + token.reserve(name.size() + 2); + token.push_back(DictKeyMark); + token.push_back(KindString); + token += name; + return token; +} + +bool NeedsEscape(const std::string& name) +{ + return !name.empty() && name.front() == DictKeyMark; +} + +// Whether a stored name is a token that has a spelling of its own, not an escaped string +bool IsSpelledToken(const std::string& stored) +{ + return stored.size() >= 2 && stored.front() == DictKeyMark && stored[1] != KindString; +} +} // namespace + +const char* UnhashableType(const InternalValue& value) +{ + return Apply(value); +} + +std::optional MakeDictKey(const InternalValue& key, const char** unhashableType) +{ + std::string token; + if (const char* type = Apply(key, token)) + { + if (unhashableType) + { + *unhashableType = type; + } + return std::nullopt; + } + return token; +} + +void ThrowUnhashable(const char* type) +{ + throw std::runtime_error(std::string("unhashable type: '") + (type ? type : "object") + "'"); +} + +std::string DictKeyText(const InternalValue& key) +{ + if (IsStringValue(key)) + { + return AsString(key); + } + std::string text; + Apply>(key, text, false); + return text; +} + +TemplateDict::TemplateDict(InternalDict&& items) + : m_items(std::move(items)) +{ +#ifndef NDEBUG + for (const auto& item : m_items) + { + assert(!NeedsEscape(item.first) && "a name never starts with the mark of a dict key token"); + } +#endif +} + +TemplateDict::TemplateDict(const TemplateDict& other) + : m_items(other.m_items) + , m_keys(other.m_keys ? std::make_unique(*other.m_keys) : nullptr) +{ +} + +TemplateDict& TemplateDict::operator=(const TemplateDict& other) +{ + if (this != &other) + { + TemplateDict copy(other); + *this = std::move(copy); + } + return *this; +} + +TemplateDict::const_iterator TemplateDict::FindString(const std::string& name) const +{ + return NeedsEscape(name) ? m_items.find(EscapeString(name)) : m_items.find(name); +} + +TemplateDict::iterator TemplateDict::FindString(const std::string& name) +{ + return NeedsEscape(name) ? m_items.find(EscapeString(name)) : m_items.find(name); +} + +TemplateDict::const_iterator TemplateDict::Find(const InternalValue& key) const +{ + auto token = MakeDictKey(key); + return token ? m_items.find(*token) : m_items.end(); +} + +TemplateDict::iterator TemplateDict::Find(const InternalValue& key) +{ + auto token = MakeDictKey(key); + return token ? m_items.find(*token) : m_items.end(); +} + +TemplateDict::iterator TemplateDict::Set(const InternalValue& key, InternalValue value) +{ + const char* unhashable = nullptr; + auto token = MakeDictKey(key, &unhashable); + if (!token) + { + ThrowUnhashable(unhashable); + } + auto [pos, inserted] = m_items.insert_or_assign(std::move(*token), std::move(value)); + if (inserted && IsSpelledToken(pos->first)) + { + try + { + if (!m_keys) + { + m_keys = std::make_unique(); + } + m_keys->emplace(pos->first, key); + } + catch (...) + { + m_items.erase(pos); + throw; + } + } + return pos; +} + +TemplateDict::iterator TemplateDict::SetString(std::string name, InternalValue value) +{ + if (NeedsEscape(name)) + { + return m_items.insert_or_assign(EscapeString(name), std::move(value)).first; + } + return m_items.insert_or_assign(std::move(name), std::move(value)).first; +} + +std::pair TemplateDict::TryEmplace(const InternalValue& key, InternalValue value) +{ + const char* unhashable = nullptr; + auto token = MakeDictKey(key, &unhashable); + if (!token) + { + ThrowUnhashable(unhashable); + } + if (auto pos = m_items.find(*token); pos != m_items.end()) + { + return { pos, false }; + } + return { Set(key, std::move(value)), true }; +} + +TemplateDict::iterator TemplateDict::Erase(const_iterator pos) +{ + if (m_keys && IsSpelledToken(pos->first)) + { + m_keys->erase(pos->first); + } + return m_items.erase(pos); +} + +void TemplateDict::Clear() +{ + m_items.clear(); + m_keys.reset(); +} + +InternalValue TemplateDict::KeyOf(const std::string& stored) const +{ + if (!NeedsEscape(stored)) + { + return InternalValue(stored); + } + if (m_keys) + { + if (auto spelling = m_keys->find(stored); spelling != m_keys->end()) + { + return spelling->second; + } + } + // An escaped string; a token without a spelling is read as the string it is + if (stored.size() >= 2 && stored[1] == KindString) + { + return InternalValue(stored.substr(2)); + } + return InternalValue(stored); +} + +} // namespace jinja2 diff --git a/src/template_dict.h b/src/template_dict.h new file mode 100644 index 000000000..6d9851e81 --- /dev/null +++ b/src/template_dict.h @@ -0,0 +1,111 @@ +#ifndef JINJA2CPP_SRC_TEMPLATE_DICT_H +#define JINJA2CPP_SRC_TEMPLATE_DICT_H + +#include "internal_value.h" + +#include +#include +#include +#include +#include + +namespace jinja2 +{ +// The dict a template owns (dict literals, dict(), namespace(), the result of the dict methods): +// Python keys, which are any hashable value, over the string keys of InternalDict (docs/tasks/0036). +// +// A string key is stored as itself. Any other hashable key is stored under a canonical token +// (MakeDictKey): one binary string that is equal for keys Python finds equal (1, 1.0 and True; +// (1, 2.0) and (1.0, 2)), so the dict needs no custom hash or comparison and a lookup is one +// find. A token cannot be turned back into a key (a float would have to be parsed), so the +// first spelling of each key is kept in a side table, allocated only when the dict has a +// key that is not a string. Iteration reads the keys from it. +// +// Every key goes in through Set or SetString; Items() is const so that no code can store a raw +// string under a token's name. + +// The first byte of every token, a byte that is not valid in UTF-8. A string that starts with +// it (host data that is not valid UTF-8) is escaped, so a token is never a plain string +inline constexpr char DictKeyMark = static_cast(0xFF); + +// The type Python's hash() rejects in a value, if any: a list or a dict, also inside a tuple. +// A range hashes, and so does an object stored as a map (namespace(), cycler()) or a host object +// (a reflected struct, which Python would hold as an object, or a JSON object) +const char* UnhashableType(const InternalValue& value); + +// The token of a key, which is equal for the keys Python treats as one. Nullopt when the key +// cannot be hashed, with the type to report in `unhashableType`: Python's TypeError +// `unhashable type: 'list'`. Objects (namespace(), cycler(), host values), undefined values and +// callables have no token of their own yet (docs/tasks/0212): they read as the empty string +std::optional MakeDictKey(const InternalValue& key, const char** unhashableType = nullptr); + +// Throws Python's TypeError for a key MakeDictKey rejects +[[noreturn]] void ThrowUnhashable(const char* type); + +// str(key): how a mapping that holds only string keys (ValuesMap) spells a key it cannot keep +std::string DictKeyText(const InternalValue& key); + +class TemplateDict +{ +public: + using iterator = InternalDict::iterator; + using const_iterator = InternalDict::const_iterator; + + TemplateDict() = default; + // A dict of string keys, such as the names of keyword arguments, which never start with the mark + explicit TemplateDict(InternalDict&& items); + TemplateDict(const TemplateDict& other); + TemplateDict(TemplateDict&& other) noexcept = default; + TemplateDict& operator=(const TemplateDict& other); + TemplateDict& operator=(TemplateDict&& other) noexcept = default; + ~TemplateDict() = default; + + // The entries under their tokens: for the count and the values. Read the keys through KeyOf + [[nodiscard]] const InternalDict& Items() const { return m_items; } + [[nodiscard]] size_t size() const { return m_items.size(); } + [[nodiscard]] bool empty() const { return m_items.empty(); } + [[nodiscard]] iterator begin() { return m_items.begin(); } + [[nodiscard]] iterator end() { return m_items.end(); } + [[nodiscard]] const_iterator begin() const { return m_items.begin(); } + [[nodiscard]] const_iterator end() const { return m_items.end(); } + // Whether some key is not a string, which the public (string keyed) view has to spell + [[nodiscard]] bool HasKeyValues() const { return m_keys && !m_keys->empty(); } + + // A string key; end() when absent + [[nodiscard]] const_iterator FindString(const std::string& name) const; + [[nodiscard]] iterator FindString(const std::string& name); + // Any key; end() when absent or unhashable (the caller checks UnhashableType where Python raises) + [[nodiscard]] const_iterator Find(const InternalValue& key) const; + [[nodiscard]] iterator Find(const InternalValue& key); + + // Python's d[key] = value: a new key is appended with this spelling, an existing key (the + // same token) keeps its position and its first spelling and takes the value. Throws + // TypeError for an unhashable key + iterator Set(const InternalValue& key, InternalValue value); + iterator SetString(std::string name, InternalValue value); + // Appends the key if it is new and keeps the value of one that exists (setdefault) + std::pair TryEmplace(const InternalValue& key, InternalValue value); + + iterator Erase(const_iterator pos); + void Clear(); + + // The key of a stored entry as a template value: the string, or the first spelling of a + // non-string key. An entry that starts with the mark and has no spelling reads as a plain string + [[nodiscard]] InternalValue KeyOf(const std::string& stored) const; + + // Dicts are equal when they hold the same entries, in any order. Tokens are canonical, so + // comparing the entries compares the keys + friend bool operator==(const TemplateDict& lhs, const TemplateDict& rhs) { return lhs.m_items == rhs.m_items; } + friend bool operator!=(const TemplateDict& lhs, const TemplateDict& rhs) { return !(lhs == rhs); } + +private: + using KeyTable = std::unordered_map; + + InternalDict m_items; + // Token to first spelling, for the entries that are not stored under their own string + std::unique_ptr m_keys; +}; + +} // namespace jinja2 + +#endif // JINJA2CPP_SRC_TEMPLATE_DICT_H diff --git a/src/testers.cpp b/src/testers.cpp index caba25367..0dcfcd276 100644 --- a/src/testers.cpp +++ b/src/testers.cpp @@ -5,6 +5,7 @@ #include "function_base.h" #include "internal_value.h" #include "render_context.h" +#include "template_dict.h" #include "undefined.h" #include "value_visitors.h" @@ -406,9 +407,21 @@ bool IsValueIn(const InternalValue& baseVal, const InternalValue& seq) } else if (seqKind == TestKind::Map) { - // `key in dict` tests the keys; dict keys are always strings here + // `key in dict` tests the keys: any hashable value for a dict the template owns, strings + // for every other map auto map = AsMap(seq); - result = map && Apply(baseVal) == TestKind::String && map->HasValue(AsString(baseVal)); + if (map && Apply(baseVal) == TestKind::String) + { + result = map->HasValue(AsString(baseVal)); + } + else if (map) + { + if (const char* type = UnhashableType(baseVal)) + { + ThrowUnhashable(type); + } + result = map->FindItem(baseVal).has_value(); + } } else if (seqKind == TestKind::String) { diff --git a/src/undefined.cpp b/src/undefined.cpp index a397e1107..a4dd69f4d 100644 --- a/src/undefined.cpp +++ b/src/undefined.cpp @@ -66,7 +66,7 @@ InternalValue MakeUndefined(const RenderContext* context, const InternalValue& o // A key a host map has but reads as undefined (a JSON null or an empty reflected field, // task 0047) is not missing: it stays a plain undefined, which fails no use auto map = AsMap(obj); - if (map && IsStringValue(key) && map->HasValue(AsString(key))) + if (map && (IsStringValue(key) ? map->HasValue(AsString(key)) : map->FindItem(key).has_value())) { return InternalValue(); } diff --git a/src/value_methods.cpp b/src/value_methods.cpp index ee80e5ac5..04da2c7a6 100644 --- a/src/value_methods.cpp +++ b/src/value_methods.cpp @@ -4,6 +4,7 @@ #include "internal_value.h" #include "markup.h" #include "render_context.h" +#include "template_dict.h" #include "unicode_tables.h" #include "value_visitors.h" @@ -50,16 +51,13 @@ bool Equals(const InternalValue& lhs, const InternalValue& rhs) return ConvertToBool(Apply2(lhs, rhs, BinaryExpression::LogicalEq)); } -// Mapping keys are strings in the value model (task 0036): other keys by their printed form -std::string KeyString(const InternalValue& key) +// Python hashes a dict key before it looks at the dict: a list or a dict is a TypeError +void CheckHashable(const InternalValue& key) { - if (IsStringValue(key)) + if (const char* type = UnhashableType(key)) { - return AsString(key); + ThrowUnhashable(type); } - std::string result; - Apply>(key, result); - return result; } // Positional argument `idx`, or the keyword argument `kwName` when the method takes one @@ -1865,7 +1863,7 @@ MapRef MapOf(const InternalValue& self) return *map; } -InternalDict& MutableDict(const InternalValue& self) +TemplateDict& MutableDict(const InternalValue& self) { auto* items = MapOf(self).GetMutableItems(); if (!items) @@ -1919,10 +1917,23 @@ InternalValue DictGet(const InternalValue& self, const CallParams& params, Rende { CheckArgs(params, "get", 1, 2); const auto& map = MapOf(self); - auto key = KeyString(params.posParams[0]); - if (map.HasValue(key)) + const auto& key = params.posParams[0]; + if (IsStringValue(key)) { - return map.GetValueByName(key); + const auto name = AsString(key); + if (map.HasValue(name)) + { + return map.GetValueByName(name); + } + } + else + { + // Python hashes the key first, also when the dict has no such key + CheckHashable(key); + if (auto found = map.FindItem(key)) + { + return std::move(*found); + } } return params.posParams.size() > 1 ? params.posParams[1] : InternalValue(NoneValue()); } @@ -1931,19 +1942,20 @@ InternalValue DictSetdefault(const InternalValue& self, const CallParams& params { CheckArgs(params, "setdefault", 1, 2); auto& items = MutableDict(self); - auto key = KeyString(params.posParams[0]); - auto p = items.find(key); + const auto& key = params.posParams[0]; + CheckHashable(key); + auto p = items.Find(key); if (p != items.end()) { return p->second; } InternalValue value = params.posParams.size() > 1 ? params.posParams[1] : InternalValue(NoneValue()); CheckNoCycle(&items, value); - items[key] = value; + items.Set(key, value); return value; } -using DictUpdates = std::vector>; +using DictUpdates = std::vector>; // The key-value pairs of an iterable of pairs, d.update([(k, v), ...]) void CollectPairUpdates(const InternalValue& other, DictUpdates& updates) @@ -1963,7 +1975,9 @@ void CollectPairUpdates(const InternalValue& other, DictUpdates& updates) { Raise("dictionary update sequence element has wrong length; 2 is required"); } - updates.emplace_back(KeyString(pair[0]), pair[1]); + // A failing update leaves the dict as it was: the keys are checked before any is stored + CheckHashable(pair[0]); + updates.emplace_back(pair[0], pair[1]); } } @@ -1975,7 +1989,7 @@ void CollectPositionalUpdates(const InternalValue& other, DictUpdates& updates) { for (auto& entry : otherMap->GetEntries()) { - updates.emplace_back(KeyString(entry.key), std::move(entry.value)); + updates.emplace_back(std::move(entry.key), std::move(entry.value)); } } else @@ -1998,16 +2012,16 @@ InternalValue DictUpdate(const InternalValue& self, const CallParams& params, Re } for (const auto& [name, value] : params.kwParams) { - updates.emplace_back(name, value); + updates.emplace_back(InternalValue(name), value); } auto& items = MutableDict(self); - for (auto& [name, value] : updates) + for (auto& [key, value] : updates) { CheckNoCycle(&items, value); } - for (auto& [name, value] : updates) + for (auto& [key, value] : updates) { - items[name] = std::move(value); + items.Set(key, std::move(value)); } return InternalValue(NoneValue()); } @@ -2016,18 +2030,21 @@ InternalValue DictPop(const InternalValue& self, const CallParams& params, Rende { CheckArgs(params, "pop", 1, 2); auto& items = MutableDict(self); - auto key = KeyString(params.posParams[0]); - auto p = items.find(key); + const auto& key = params.posParams[0]; + CheckHashable(key); + auto p = items.Find(key); if (p == items.end()) { if (params.posParams.size() > 1) { return params.posParams[1]; } - Raise("KeyError: '" + key + "'"); + std::string spelling; + Apply>(key, spelling, true); + Raise("KeyError: " + spelling); } auto result = p->second; - items.erase(p); + items.Erase(p); return result; } @@ -2040,8 +2057,8 @@ InternalValue DictPopitem(const InternalValue& self, const CallParams& params, R Raise("popitem(): dictionary is empty"); } auto last = std::prev(items.end()); - auto result = MakeTuple({ InternalValue(last->first), last->second }); - items.erase(last); + auto result = MakeTuple({ items.KeyOf(last->first), last->second }); + items.Erase(last); return result; } @@ -2049,10 +2066,14 @@ InternalValue DictCopy(const InternalValue& self, const CallParams& params, Rend { CheckArgs(params, "copy", 0, 0); const auto& map = MapOf(self); - InternalDict result; + if (const auto* items = map.GetMutableItems()) + { + return CreateMapAdapter(TemplateDict(*items)); + } + TemplateDict result; for (auto& entry : map.GetEntries()) { - result[KeyString(entry.key)] = std::move(entry.value); + result.Set(entry.key, std::move(entry.value)); } return CreateMapAdapter(std::move(result)); } @@ -2060,7 +2081,7 @@ InternalValue DictCopy(const InternalValue& self, const CallParams& params, Rend InternalValue DictClear(const InternalValue& self, const CallParams& params, RenderContext&) { CheckArgs(params, "clear", 0, 0); - MutableDict(self).clear(); + MutableDict(self).Clear(); return InternalValue(NoneValue()); } @@ -2256,7 +2277,7 @@ void StoreItem(const InternalValue& container, const InternalValue& key, Interna auto* items = map->GetMutableItems(); if (items) { - (*items)[KeyString(key)] = std::move(value); + items->Set(key, std::move(value)); } } } @@ -2305,14 +2326,14 @@ InternalValue MakeMutable(const InternalValue& value) } if (auto map = AsMap(value)) { - InternalDict items; + TemplateDict items; for (auto& [key, item] : map->GetEntries()) { if (extendLifetime) { item.SetParentData(value); } - items[KeyString(key)] = std::move(item); + items.Set(key, std::move(item)); } return CreateMapAdapter(std::move(items)); } diff --git a/test/internal_value_test.cpp b/test/internal_value_test.cpp index c9a5fb83c..d939cae64 100644 --- a/test/internal_value_test.cpp +++ b/test/internal_value_test.cpp @@ -1,4 +1,5 @@ #include "../src/internal_value.h" +#include "../src/template_dict.h" #include "../src/value_visitors.h" #include "gtest/gtest.h" @@ -9,12 +10,15 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include using namespace jinja2; @@ -225,4 +229,152 @@ TEST(InternalValueLayoutTest, CopiesOfAGeneratorIterateApart) EXPECT_EQ(anchored, copy.ShouldExtendLifetime()); } } + +// Dict keys follow Python's equality (docs/tasks/0036): the token is the same for keys that are +// one key +TEST(TemplateDictTest, KeyTokensFollowPythonEquality) +{ + const auto token = [](const InternalValue& key) { return MakeDictKey(key); }; + const auto tuple = [](InternalValueList items) { return InternalValue(ListAdapter::CreateAdapter(std::move(items)).MarkAsTuple()); }; + const InternalValue one(int64_t{ 1 }); + + ASSERT_TRUE(token(one)); + EXPECT_EQ(token(one), token(InternalValue(1.0))); + EXPECT_EQ(token(one), token(InternalValue(true))); + EXPECT_NE(token(one), token(InternalValue(1.5))); + EXPECT_NE(token(one), token(InternalValue(int64_t{ 2 }))); + EXPECT_EQ(token(InternalValue(int64_t{ 0 })), token(InternalValue(-0.0))); + EXPECT_EQ(token(InternalValue(false)), token(InternalValue(0.0))); + // A float beyond int64 is still a float + EXPECT_NE(token(InternalValue(9223372036854775808.0)), token(InternalValue(std::numeric_limits::max()))); + + EXPECT_EQ(token(tuple({ one, InternalValue(2.0) })), token(tuple({ InternalValue(1.0), InternalValue(int64_t{ 2 }) }))); + EXPECT_NE(token(tuple({ one, InternalValue(int64_t{ 2 }) })), token(tuple({ InternalValue(int64_t{ 2 }), one }))); + // (1, 2) is not the nested ((1), 2) nor the longer (1, 2, 3) + EXPECT_NE(token(tuple({ one, InternalValue(int64_t{ 2 }) })), token(tuple({ one, InternalValue(int64_t{ 2 }), one }))); + EXPECT_EQ(token(tuple({ tuple({ one }), InternalValue(std::string("a")) })), token(tuple({ tuple({ InternalValue(1.0) }), InternalValue(std::wstring(L"a")) }))); + + EXPECT_EQ(token(InternalValue(std::string("a"))), token(InternalValue(std::wstring(L"a")))); + EXPECT_EQ(std::string("a"), *token(InternalValue(std::string("a")))); + // A string that spells the token of the integer 1 is another key + EXPECT_NE(token(one), token(InternalValue(*token(one)))); + EXPECT_NE(token(InternalValue(NoneValue{})), token(InternalValue(std::string("N")))); + + const auto range = [](int64_t start, int64_t stop, int64_t step) { return InternalValue(ListAdapter::CreateRange(start, stop, step)); }; + // Equal ranges are equal keys: range(0, 3, 2) and range(0, 4, 2) are both [0, 2] + EXPECT_EQ(token(range(0, 3, 2)), token(range(0, 4, 2))); + EXPECT_EQ(token(range(5, 5, 1)), token(range(0, 0, 3))); + EXPECT_NE(token(range(0, 3, 1)), token(range(0, 4, 2))); +} + +TEST(TemplateDictTest, UnhashableKeys) +{ + const char* type = nullptr; + const InternalValue list(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(int64_t{ 1 }) })); + EXPECT_FALSE(MakeDictKey(list, &type)); + EXPECT_STREQ("list", type); + + const InternalValue tupleWithList(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(int64_t{ 1 }), list }).MarkAsTuple()); + type = nullptr; + EXPECT_FALSE(MakeDictKey(tupleWithList, &type)); + EXPECT_STREQ("list", type); + + const InternalValue dict(CreateMapAdapter(TemplateDict())); + type = nullptr; + EXPECT_FALSE(MakeDictKey(dict, &type)); + EXPECT_STREQ("dict", type); + + TemplateDict target; + EXPECT_THROW(target.Set(list, InternalValue(int64_t{ 1 })), std::runtime_error); + EXPECT_TRUE(target.empty()); + EXPECT_EQ(target.end(), target.Find(list)); +} + +TEST(TemplateDictTest, FirstKeyAndPositionKeptLastValueWins) +{ + TemplateDict dict; + dict.Set(InternalValue(int64_t{ 1 }), InternalValue(std::string("a"))); + dict.Set(InternalValue(std::string("x")), InternalValue(std::string("b"))); + dict.Set(InternalValue(1.0), InternalValue(std::string("c"))); + dict.Set(InternalValue(true), InternalValue(std::string("d"))); + + ASSERT_EQ(2U, dict.size()); + EXPECT_TRUE(dict.HasKeyValues()); + auto first = dict.begin(); + EXPECT_EQ(ValueKind::Int, dict.KeyOf(first->first).Kind()); + EXPECT_EQ("d", AsString(first->second)); + EXPECT_EQ(ValueKind::String, dict.KeyOf(std::next(first)->first).Kind()); + + auto found = dict.Find(InternalValue(1.0)); + ASSERT_NE(dict.end(), found); + EXPECT_EQ("d", AsString(found->second)); + dict.Erase(found); + EXPECT_FALSE(dict.HasKeyValues()); + EXPECT_EQ(1U, dict.size()); + + // A copy has keys of its own + dict.Set(InternalValue(2.5), InternalValue(int64_t{ 1 })); + TemplateDict copy(dict); + copy.Set(InternalValue(2.5), InternalValue(int64_t{ 2 })); + copy.Set(InternalValue(3.5), InternalValue(int64_t{ 3 })); + EXPECT_EQ(2U, dict.size()); + EXPECT_EQ(3U, copy.size()); + const auto key = dict.KeyOf(dict.Find(InternalValue(2.5))->first); + ASSERT_NE(nullptr, GetIf(&key)); + EXPECT_EQ(2.5, *GetIf(&key)); + EXPECT_FALSE(dict == copy); +} + +TEST(TemplateDictTest, StringKeysStartingWithTheMarkAreEscaped) +{ + TemplateDict dict; + const std::string odd = std::string(1, DictKeyMark) + "n12345678"; + dict.SetString(odd, InternalValue(int64_t{ 7 })); + dict.Set(InternalValue(int64_t{ 1 }), InternalValue(int64_t{ 8 })); + + ASSERT_EQ(2U, dict.size()); + ASSERT_NE(dict.end(), dict.FindString(odd)); + EXPECT_EQ(7, ConvertToInt(dict.FindString(odd)->second)); + ASSERT_NE(dict.end(), dict.Find(InternalValue(odd))); + // Read back as the string it was + EXPECT_EQ(odd, AsString(dict.KeyOf(dict.begin()->first))); + EXPECT_EQ(ValueKind::Int, dict.KeyOf(std::next(dict.begin())->first).Kind()); +} + +TEST(TemplateDictTest, PublicMapSpellsKeys) +{ + TemplateDict dict; + dict.Set(InternalValue(int64_t{ 1 }), InternalValue(std::string("x"))); + dict.SetString("b", InternalValue(int64_t{ 2 })); + dict.Set(InternalValue(ListAdapter::CreateAdapter(InternalValueList{ InternalValue(int64_t{ 1 }), InternalValue(int64_t{ 2 }) }).MarkAsTuple()), InternalValue(int64_t{ 3 })); + dict.Set(InternalValue(std::string("1")), InternalValue(std::string("y"))); + + const auto map = CreateMapAdapter(std::move(dict)); + const auto value = IntValue2Value(InternalValue(map)); + const auto generic = value.ToGenericMap(); + ASSERT_NE(nullptr, generic.GetAccessor()); + // Later value wins at the position of the first spelling + EXPECT_EQ((std::vector{ "1", "b", "(1, 2)" }), generic.GetKeys()); + EXPECT_EQ("y", generic.GetValueByName("1").ToString()); + EXPECT_EQ(3U, generic.GetSize()); + // A copy: later changes of the template's dict do not reach it + map.GetMutableItems()->SetString("c", InternalValue(int64_t{ 4 })); + EXPECT_EQ(3U, generic.GetSize()); + + // A dict of string keys stays a live view + TemplateDict strings; + strings.SetString("a", InternalValue(int64_t{ 1 })); + const auto stringMap = CreateMapAdapter(std::move(strings)); + const auto stringGeneric = IntValue2Value(InternalValue(stringMap)).ToGenericMap(); + EXPECT_EQ(1U, stringGeneric.GetSize()); + stringMap.GetMutableItems()->SetString("b", InternalValue(int64_t{ 2 })); + EXPECT_EQ(2U, stringGeneric.GetSize()); +} + +TEST(TemplateDictTest, DictsCompareAsPythonDicts) +{ + Template tpl; + ASSERT_TRUE(tpl.Load("{{ {1: 'a', (1, 2): 'b'} == {(1.0, 2): 'b', True: 'a'} }}|{{ {1: 'a'} == {'1': 'a'} }}|{{ 1 in {True: 0} }}").has_value()); + EXPECT_EQ("True|False|True", tpl.RenderAsString({}).value()); +} #endif diff --git a/test/parity/cases/errors.py b/test/parity/cases/errors.py index f8c7e3ea9..ea91bba18 100644 --- a/test/parity/cases/errors.py +++ b/test/parity/cases/errors.py @@ -102,4 +102,10 @@ # caller() outside a call block raises "No caller defined" (docs/tasks/0169) ("caller_without_call_block", "{% macro m() %}[{{ caller() }}]{% endmacro %}{{ m() }}"), ("call_body_reads_own_missing_caller", "{% macro inner() %}<{{ caller() }}>{% endmacro %}{% macro m() %}{% call inner() %}{{ caller() }}{% endcall %}{% endmacro %}{% call m() %}x{% endcall %}"), + ('dict_unhashable_dict_key', "{{ {{'a':1}: 2} }}"), + ('dict_unhashable_tuple_key', '{{ {(1, [2]): 3} }}'), + ('in_dict_unhashable', '{{ [1] in {1: 2} }}'), + ('dict_get_unhashable', '{{ {1: 2}.get([1]) }}'), + ('dict_setdefault_unhashable', '{{ {1: 2}.setdefault([1], 3) }}'), + ('dict_pop_unhashable', '{{ {1: 2}.pop([1]) }}'), ] diff --git a/test/parity/cases/filters.py b/test/parity/cases/filters.py index 562eeb3d4..1dca2ee72 100644 --- a/test/parity/cases/filters.py +++ b/test/parity/cases/filters.py @@ -329,4 +329,23 @@ ("truncate_int", "{{ 5|truncate }}"), ("truncate_undefined_checks_length", "{{ missing|truncate(1, leeway=0) }}"), ("truncate_end_not_string", "{{ 'abc'|truncate(1, leeway=0, end=1) }}"), + ('list_dict_int_keys', "{{ {2: 'a', 1: 'b'}|list }}|{{ {2: 'a', 1.5: 'b', None: 'c'}|list }}|{{ {1: 'a', 2: 'b'}|first }}|{{ {1: 'a', 2: 'b'}|length }}"), + ('dictsort_numeric_keys', "{{ {10: 'a', 2: 'b', 2.5: 'c', True: 'd'}|dictsort }}"), + ('dictsort_mixed_key_types', "{{ {'b': 1, 1: 2}|dictsort }}"), + ('dictsort_none_keys', '{{ {None: 1, 2: 2}|dictsort }}'), + ('dictsort_single_none_key', '{{ {None: 1}|dictsort }}'), + ('dictsort_reverse_numeric', "{{ {1: 'a', 3: 'c', 2: 'b'}|dictsort(reverse=true) }}"), + ('tojson_non_string_keys', "{{ {1: 'a', 2.5: 'b'}|tojson }}|{{ {None: 1}|tojson }}|{{ {True: 1}|tojson }}"), + ('tojson_key_order', "{{ {10: 'a', 2: 'b', 1.5: 'c', False: 'd'}|tojson }}"), + ('tojson_tuple_key', '{{ {(1, 2): 1}|tojson }}'), + ('tojson_mixed_key_types', "{{ {'a': 1, 2: 2}|tojson }}"), + ('tojson_none_among_keys', "{{ {None: 1, 'a': 2}|tojson }}"), + ('xmlattr_int_key', "{{ {1: 'a'}|xmlattr }}"), + ('xmlattr_int_key_none_value', '[{{ {1: none}|xmlattr }}]'), + ('urlencode_int_key', '{{ {1: 2}|urlencode }}'), + ('pprint_non_string_keys', "{{ {10: 2, 2: 1, 'a': 3}|pprint }}|{{ {(1, 2): 3}|pprint }}"), + ('pprint_mixed_type_keys', "{{ {'b': 1, 2.5: 2, None: 3, 1: 4, (1, 2): 5, True: 6}|pprint }}"), + ('string_dict_non_string_keys', "{{ {1: 'a', None: 'b', (1, 2): 'c'}|string }}"), + ('format_percent_int_key', "{{ '%(1)s'|format(a=1) }}"), + ('map_over_dict_int_keys', "{{ {1: 'a', 2: 'b'}|map('string')|join(',') }}|{{ {1: 'a', 2: 'b'}|sum }}"), ] diff --git a/test/parity/cases/globals.py b/test/parity/cases/globals.py index 7c1a1f7d4..7c867d6f3 100644 --- a/test/parity/cases/globals.py +++ b/test/parity/cases/globals.py @@ -77,4 +77,8 @@ ("lipsum_no_paragraphs", "[{{ lipsum(0, min=200) }}]"), # an object can be a dict key: Python hashes it by identity (0212) ("object_dict_key", "{{ {namespace(): 1} }}"), + ('dict_from_non_string_pairs', "{{ dict([(1, 'a'), ((1, 2), 'b')]) }}|{{ dict({1: 'a'}, b=2) }}|{{ namespace({1: 2}) }}"), + ('dict_unhashable_pair_key', '{{ dict([([1], 2)]) }}'), + ('dict_merge_equal_numeric_keys', "{{ dict([(1, 'a'), (1.0, 'b'), (True, 'c')]) }}"), + ('namespace_non_string_subscript', '{% set ns = namespace(a=1) %}[{{ ns[1] }}][{{ ns.a }}]'), ] diff --git a/test/parity/cases/literals.py b/test/parity/cases/literals.py index 9da45a500..4d173d22b 100644 --- a/test/parity/cases/literals.py +++ b/test/parity/cases/literals.py @@ -68,4 +68,11 @@ ("dict_nested_lookup", "{% set d = {'a': {'b': [1, 2]}} %}{{ d['a']['b'][1] }}"), ("parenthesised", "{{ (1) }}"), ("number_dot_attribute", "[{{ 1.e3 }}][{{ 1.5.2 }}]"), + ('dict_int_key_print', "{{ {1: 'x', '1': 'y'} }}"), + ('dict_numeric_keys_merge', "{{ {1:'a', 1.0:'b', True:'c'} }}|{{ {1.0:'x'} }}|{{ {True:1}[1] }}|{{ {0:'z'}[false] }}"), + ('dict_other_hashable_keys', '{{ {none:1, 1.5:2, (1,2):3} }}|{{ {(1,2):3}[(1,2)] }}|{{ {(1,2.0):3}[(1.0,2)] }}|{{ {none:1}[none] }}'), + ('dict_int_key_not_string', "{{ {1:'x'}['1'] is defined }}|{{ {'1':'x'}[1] is defined }}|{{ {1:2}.1 }}|{{ {-1:2}[-1] }}"), + ('dict_float_keys', "{{ {0.1+0.2:'x'} }}|{{ {-0.0:'x', 0:'y'} }}|{{ {1e100:'x'} }}"), + ('dict_key_range_and_nested_tuple', "{{ {range(3): 'r', ((1, 2), 'a'): 'n'} }}|{{ {range(3): 'r'}[range(0, 3)] }}|{{ {((1, 2), 'a'): 'n'}[((1.0, 2), 'a')] }}"), + ('dict_key_big_float_is_float', "{{ {2.0**63: 'f'}[2.0**63] }}|{{ {2.0**63: 'f'} }}"), ] diff --git a/test/parity/cases/methods.py b/test/parity/cases/methods.py index c1e40c5fc..e9ebbe789 100644 --- a/test/parity/cases/methods.py +++ b/test/parity/cases/methods.py @@ -99,4 +99,13 @@ ("str_format_huge_width", "{{ '{:99999999999999999999}'.format(1) }}"), ("loop_cycle_no_args", "{% for x in [1, 2] %}{{ loop.cycle() }}{% endfor %}"), ("loop_attr_not_dict_method", "{% for i in [1] %}{{ loop.items is defined }}{% endfor %}"), + ('dict_methods_non_string_keys', "{% set d = {1: 'a'} %}{{ d.get(1) }}|{{ d.get(1.0) }}|{{ d.get('1', 'none') }}|{{ d.keys()|list }}|{{ d.setdefault(2.0, 'b') }}|{% do d.update({(1, 2): 'c'}) %}{{ d }}|{{ d.pop(1) }}|{{ d.popitem() }}|{{ d }}", {"env": {"extensions": ["do"]}}), + ('dict_get_host_int_key', "{{ dd.get(1) }}|{{ dd.get(1, 'dflt') }}", {"ctx": {"dd": {"1": 2}}}), + ('dict_pop_missing_int', '{{ {}.pop(1) }}'), + ('format_index_int_key', "{{ '{0[1]}'.format({1: 2}) }}"), + ('dict_update_pairs_and_kwargs', "{% set d = {1: 'a'} %}{% do d.update([(1.0, 'b'), (None, 'n')], z=1) %}{{ d }}", {"env": {"extensions": ["do"]}}), + ('dict_copy_keeps_keys', "{% set d = {1: 'a', (2, 3): 'b'} %}{% set c = d.copy() %}{% do c.update({1: 'x'}) %}{{ d }}|{{ c }}", {"env": {"extensions": ["do"]}}), + ('dict_setdefault_existing_float', "{% set d = {1: 'a'} %}{{ d.setdefault(1.0, 'z') }}{{ d }}", {"env": {"extensions": ["do"]}}), + ('dict_update_unhashable_unchanged', "{% set d = {1: 'a'} %}{% do d.update([(2, 'b'), ([3], 'c')]) %}", {"env": {"extensions": ["do"]}}), + ('dict_store_item_non_string', "{% set d = {1: 'a'} %}{% do d.update({2: 'b'}) %}{{ d[2] }}{{ d.get(2) }}{{ d|length }}", {"env": {"extensions": ["do"]}}), ] diff --git a/test/parity/cases/operators.py b/test/parity/cases/operators.py index 636b894e0..1deaa05e8 100644 --- a/test/parity/cases/operators.py +++ b/test/parity/cases/operators.py @@ -123,4 +123,7 @@ ("precedence_compare_operands", "{{ 'T' if x + 1 == 2 * 2 > x - 1 else 'F' }}|{{ 'T' if x - 3 not in l and 2 in l else 'F' }}"), ("precedence_unary_filter", "{{ -x|abs + 1 }}|{{ - - x }}|{{ 2 ** -x|abs }}"), ("not_after_binary_operator", "{{ x + not z }}"), + ('in_dict_non_string_key', "{{ 'T' if 1 in {1: 'a'} else 'F' }}|{{ 'T' if '1' in {1: 'a'} else 'F' }}|{{ 'T' if 1.0 in {1: 'a'} else 'F' }}|{{ 'T' if (1, 2) in {(1, 2): 'a'} else 'F' }}|{{ 'T' if none in {none: 'a'} else 'F' }}"), + ('eq_dict_non_string_keys', "{{ 'T' if {1: 'a'} == {1.0: 'a'} else 'F' }}|{{ 'T' if {1: 'a'} == {'1': 'a'} else 'F' }}|{{ 'T' if {(1, 2): 3} == {(1.0, 2): 3} else 'F' }}"), + ('not_in_dict_non_string_key', "{{ 'T' if 2 not in {1: 'a'} else 'F' }}|{{ 'T' if True in {1: 'a'} else 'F' }}"), ] diff --git a/test/parity/cases/statements.py b/test/parity/cases/statements.py index 9b8fe9e56..e21a9c864 100644 --- a/test/parity/cases/statements.py +++ b/test/parity/cases/statements.py @@ -435,4 +435,6 @@ # the else frame binds at entry, and its set hides a loop target outside only there (docs/tasks/0202) ('idtracking_loop_else_binds_at_entry', '{% for i in e %}{% else %}{% for k in [1] %}{{ x }}{% endfor %}{% set x = 1 %}{% endfor %}'), ('loop_else_set_keeps_outer_target', '{% for x in [1] %}{% for j in e %}{% else %}{% set x = 2 %}{{ x }}{% endfor %}{{ x }}{% endfor %}'), + ('for_dict_int_keys', "{% for k in {1: 'a', 2: 'b'} %}{{ k + 1 }}{% endfor %}|{% for k, v in {2: 'a'}.items() %}{{ k }}{{ v }}{% endfor %}|{% for k, v in {20: 'x', 10: 'y'}|dictsort %}{{ k }}{% endfor %}"), + ('for_dict_tuple_key_unpack', "{% for k, v in {(1, 2): 'a'}.items() %}{{ k[0] }}{{ k[1] }}{{ v }}{% endfor %}"), ] diff --git a/test/parity/cases/subscripts.py b/test/parity/cases/subscripts.py index 6949a7eab..e3ace2eb1 100644 --- a/test/parity/cases/subscripts.py +++ b/test/parity/cases/subscripts.py @@ -41,4 +41,7 @@ ("dot_index_chain", "{{ d.b.c }}{{ users.1.name }}"), ("slice_extreme_step", "{{ l[1::9223372036854775807] }}|{{ l[::-9223372036854775807-1] }}"), ("tuple_index", "[{{ l[1, 2] }}][{{ l[] }}]"), + ('dict_unhashable_subscript', '[{{ {1:2}[[1]] }}]'), + ('dict_subscript_float_and_bool', "{{ {1: 'a'}[1.0] }}|{{ {1: 'a'}[true] }}|{{ {1.5: 'f'}[1.5] }}"), + ('dict_subscript_missing_non_string', "[{{ {1: 'a'}[2] }}][{{ {1: 'a'}[none] }}]"), ] diff --git a/test/parity/divergences/errors.txt b/test/parity/divergences/errors.txt index d869f3e9b..cc82b4d1c 100644 --- a/test/parity/divergences/errors.txt +++ b/test/parity/divergences/errors.txt @@ -1,7 +1,6 @@ # Known divergences of Jinja2C++ from Python Jinja2 in the errors area (cases/errors.py). # Format and kinds: test/parity/README.md. -errors.dict_unhashable_key accepts 0036 unhashable dict keys are accepted (keys are stringified) errors.if_test_conditional_expression accepts 0213 an if test takes a conditional expression; Jinja2 parses it without one errors.raw_unclosed_at_end rejects 0213 an unclosed raw block at the end of the template; Jinja2 renders it as empty errors.raw_unclosed_minus_space rejects 0213 an unclosed raw block at the end of the template; Jinja2 renders it as empty diff --git a/test/parity/divergences/literals.txt b/test/parity/divergences/literals.txt index 936bb63c5..6551c7b3d 100644 --- a/test/parity/divergences/literals.txt +++ b/test/parity/divergences/literals.txt @@ -1,6 +1,5 @@ # Known divergences of Jinja2C++ from Python Jinja2 in the literals area (cases/literals.py). # Format and kinds: test/parity/README.md. -literals.dict_int_key output 0036 mapping keys are strings; an integer key cannot be looked up by integer literals.int_big output 0015 deliberate: integers are 64-bit; a literal beyond int64 becomes a float (docs/parity.md) literals.string_escape_named rejects 0041 \N{name} needs the Unicode name table; it is rejected as malformed (documented divergence) diff --git a/test/parity/expected/errors.json b/test/parity/expected/errors.json index 7b917cc32..60ad7370f 100644 --- a/test/parity/expected/errors.json +++ b/test/parity/expected/errors.json @@ -1843,6 +1843,120 @@ "type": "UndefinedError", "message": "No caller defined" } + }, + { + "id": "errors.dict_unhashable_dict_key", + "template": "{{ {{'a':1}: 2} }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'dict'" + } + }, + { + "id": "errors.dict_unhashable_tuple_key", + "template": "{{ {(1, [2]): 3} }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.in_dict_unhashable", + "template": "{{ [1] in {1: 2} }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.dict_get_unhashable", + "template": "{{ {1: 2}.get([1]) }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.dict_setdefault_unhashable", + "template": "{{ {1: 2}.setdefault([1], 3) }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "errors.dict_pop_unhashable", + "template": "{{ {1: 2}.pop([1]) }}", + "context": { + "x": 3, + "l": [ + 1, + 2 + ], + "s": "abc" + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } } ] } diff --git a/test/parity/expected/filters.json b/test/parity/expected/filters.json index b6681a1ec..71a5d476d 100644 --- a/test/parity/expected/filters.json +++ b/test/parity/expected/filters.json @@ -20519,6 +20519,1288 @@ "type": "TypeError", "message": "object of type 'int' has no len()" } + }, + { + "id": "filters.list_dict_int_keys", + "template": "{{ {2: 'a', 1: 'b'}|list }}|{{ {2: 'a', 1.5: 'b', None: 'c'}|list }}|{{ {1: 'a', 2: 'b'}|first }}|{{ {1: 'a', 2: 'b'}|length }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[2, 1]|[2, 1.5, None]|1|2" + }, + { + "id": "filters.dictsort_numeric_keys", + "template": "{{ {10: 'a', 2: 'b', 2.5: 'c', True: 'd'}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[(True, 'd'), (2, 'b'), (2.5, 'c'), (10, 'a')]" + }, + { + "id": "filters.dictsort_mixed_key_types", + "template": "{{ {'b': 1, 1: 2}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'int' and 'str'" + } + }, + { + "id": "filters.dictsort_none_keys", + "template": "{{ {None: 1, 2: 2}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'int' and 'NoneType'" + } + }, + { + "id": "filters.dictsort_single_none_key", + "template": "{{ {None: 1}|dictsort }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[(None, 1)]" + }, + { + "id": "filters.dictsort_reverse_numeric", + "template": "{{ {1: 'a', 3: 'c', 2: 'b'}|dictsort(reverse=true) }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[(3, 'c'), (2, 'b'), (1, 'a')]" + }, + { + "id": "filters.tojson_non_string_keys", + "template": "{{ {1: 'a', 2.5: 'b'}|tojson }}|{{ {None: 1}|tojson }}|{{ {True: 1}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{\"1\": \"a\", \"2.5\": \"b\"}|{\"null\": 1}|{\"true\": 1}" + }, + { + "id": "filters.tojson_key_order", + "template": "{{ {10: 'a', 2: 'b', 1.5: 'c', False: 'd'}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{\"false\": \"d\", \"1.5\": \"c\", \"2\": \"b\", \"10\": \"a\"}" + }, + { + "id": "filters.tojson_tuple_key", + "template": "{{ {(1, 2): 1}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "keys must be str, int, float, bool or None, not tuple" + } + }, + { + "id": "filters.tojson_mixed_key_types", + "template": "{{ {'a': 1, 2: 2}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'int' and 'str'" + } + }, + { + "id": "filters.tojson_none_among_keys", + "template": "{{ {None: 1, 'a': 2}|tojson }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "'<' not supported between instances of 'str' and 'NoneType'" + } + }, + { + "id": "filters.xmlattr_int_key", + "template": "{{ {1: 'a'}|xmlattr }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "expected string or bytes-like object, got 'int'" + } + }, + { + "id": "filters.xmlattr_int_key_none_value", + "template": "[{{ {1: none}|xmlattr }}]", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "[]" + }, + { + "id": "filters.urlencode_int_key", + "template": "{{ {1: 2}|urlencode }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "1=2" + }, + { + "id": "filters.pprint_non_string_keys", + "template": "{{ {10: 2, 2: 1, 'a': 3}|pprint }}|{{ {(1, 2): 3}|pprint }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{2: 1, 10: 2, 'a': 3}|{(1, 2): 3}" + }, + { + "id": "filters.pprint_mixed_type_keys", + "template": "{{ {'b': 1, 2.5: 2, None: 3, 1: 4, (1, 2): 5, True: 6}|pprint }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{None: 3, 1: 6, 2.5: 2, 'b': 1, (1, 2): 5}" + }, + { + "id": "filters.string_dict_non_string_keys", + "template": "{{ {1: 'a', None: 'b', (1, 2): 'c'}|string }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "{1: 'a', None: 'b', (1, 2): 'c'}" + }, + { + "id": "filters.format_percent_int_key", + "template": "{{ '%(1)s'|format(a=1) }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "KeyError", + "message": "'1'" + } + }, + { + "id": "filters.map_over_dict_int_keys", + "template": "{{ {1: 'a', 2: 'b'}|map('string')|join(',') }}|{{ {1: 'a', 2: 'b'}|sum }}", + "context": { + "x": 3, + "neg": -3, + "y": 2.567, + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "dup": [ + 3, + 1, + 3, + 2, + 1 + ], + "words": [ + "b", + "A", + "c" + ], + "d": { + "b": 2, + "a": 1, + "C": 3 + }, + "e": [], + "n": null, + "html": "Tom & \"Jerry\"", + "long": "The quick brown fox jumps over the lazy dog", + "users": [ + { + "name": "bob", + "age": 30, + "city": "Oslo" + }, + { + "name": "alice", + "age": 25, + "city": "Rome" + }, + { + "name": "carl", + "age": 30, + "city": "Oslo" + }, + { + "name": "dan", + "age": 41 + } + ], + "lines": "a\nb\n\nc", + "nums_str": [ + "10", + "9", + "100" + ] + }, + "env": {}, + "templates": {}, + "output": "1,2|3" } ] } diff --git a/test/parity/expected/globals.json b/test/parity/expected/globals.json index a9d6ba757..385a9433a 100644 --- a/test/parity/expected/globals.json +++ b/test/parity/expected/globals.json @@ -705,6 +705,42 @@ "env": {}, "templates": {}, "output": "{: 1}" + }, + { + "id": "globals.dict_from_non_string_pairs", + "template": "{{ dict([(1, 'a'), ((1, 2), 'b')]) }}|{{ dict({1: 'a'}, b=2) }}|{{ namespace({1: 2}) }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'a', (1, 2): 'b'}|{1: 'a', 'b': 2}|" + }, + { + "id": "globals.dict_unhashable_pair_key", + "template": "{{ dict([([1], 2)]) }}", + "context": {}, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "globals.dict_merge_equal_numeric_keys", + "template": "{{ dict([(1, 'a'), (1.0, 'b'), (True, 'c')]) }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'c'}" + }, + { + "id": "globals.namespace_non_string_subscript", + "template": "{% set ns = namespace(a=1) %}[{{ ns[1] }}][{{ ns.a }}]", + "context": {}, + "env": {}, + "templates": {}, + "output": "[][1]" } ] } diff --git a/test/parity/expected/literals.json b/test/parity/expected/literals.json index 7d5a0c034..fd5c9ed2b 100644 --- a/test/parity/expected/literals.json +++ b/test/parity/expected/literals.json @@ -569,6 +569,62 @@ "env": {}, "templates": {}, "output": "[][]" + }, + { + "id": "literals.dict_int_key_print", + "template": "{{ {1: 'x', '1': 'y'} }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'x', '1': 'y'}" + }, + { + "id": "literals.dict_numeric_keys_merge", + "template": "{{ {1:'a', 1.0:'b', True:'c'} }}|{{ {1.0:'x'} }}|{{ {True:1}[1] }}|{{ {0:'z'}[false] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{1: 'c'}|{1.0: 'x'}|1|z" + }, + { + "id": "literals.dict_other_hashable_keys", + "template": "{{ {none:1, 1.5:2, (1,2):3} }}|{{ {(1,2):3}[(1,2)] }}|{{ {(1,2.0):3}[(1.0,2)] }}|{{ {none:1}[none] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{None: 1, 1.5: 2, (1, 2): 3}|3|3|1" + }, + { + "id": "literals.dict_int_key_not_string", + "template": "{{ {1:'x'}['1'] is defined }}|{{ {'1':'x'}[1] is defined }}|{{ {1:2}.1 }}|{{ {-1:2}[-1] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "False|False|2|2" + }, + { + "id": "literals.dict_float_keys", + "template": "{{ {0.1+0.2:'x'} }}|{{ {-0.0:'x', 0:'y'} }}|{{ {1e100:'x'} }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{0.30000000000000004: 'x'}|{-0.0: 'y'}|{1e+100: 'x'}" + }, + { + "id": "literals.dict_key_range_and_nested_tuple", + "template": "{{ {range(3): 'r', ((1, 2), 'a'): 'n'} }}|{{ {range(3): 'r'}[range(0, 3)] }}|{{ {((1, 2), 'a'): 'n'}[((1.0, 2), 'a')] }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "{range(0, 3): 'r', ((1, 2), 'a'): 'n'}|r|n" + }, + { + "id": "literals.dict_key_big_float_is_float", + "template": "{{ {2.0**63: 'f'}[2.0**63] }}|{{ {2.0**63: 'f'} }}", + "context": {}, + "env": {}, + "templates": {}, + "output": "f|{9.223372036854776e+18: 'f'}" } ] } diff --git a/test/parity/expected/methods.json b/test/parity/expected/methods.json index b632c70bc..4f402ca05 100644 --- a/test/parity/expected/methods.json +++ b/test/parity/expected/methods.json @@ -2029,6 +2029,218 @@ "env": {}, "templates": {}, "output": "False" + }, + { + "id": "methods.dict_methods_non_string_keys", + "template": "{% set d = {1: 'a'} %}{{ d.get(1) }}|{{ d.get(1.0) }}|{{ d.get('1', 'none') }}|{{ d.keys()|list }}|{{ d.setdefault(2.0, 'b') }}|{% do d.update({(1, 2): 'c'}) %}{{ d }}|{{ d.pop(1) }}|{{ d.popitem() }}|{{ d }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "a|a|none|[1]|b|{1: 'a', 2.0: 'b', (1, 2): 'c'}|a|((1, 2), 'c')|{2.0: 'b'}" + }, + { + "id": "methods.dict_get_host_int_key", + "template": "{{ dd.get(1) }}|{{ dd.get(1, 'dflt') }}", + "context": { + "dd": { + "1": 2 + } + }, + "env": {}, + "templates": {}, + "output": "None|dflt" + }, + { + "id": "methods.dict_pop_missing_int", + "template": "{{ {}.pop(1) }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": {}, + "templates": {}, + "error": { + "phase": "render", + "type": "KeyError", + "message": "1" + } + }, + { + "id": "methods.format_index_int_key", + "template": "{{ '{0[1]}'.format({1: 2}) }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": {}, + "templates": {}, + "output": "2" + }, + { + "id": "methods.dict_update_pairs_and_kwargs", + "template": "{% set d = {1: 'a'} %}{% do d.update([(1.0, 'b'), (None, 'n')], z=1) %}{{ d }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "{1: 'b', None: 'n', 'z': 1}" + }, + { + "id": "methods.dict_copy_keeps_keys", + "template": "{% set d = {1: 'a', (2, 3): 'b'} %}{% set c = d.copy() %}{% do c.update({1: 'x'}) %}{{ d }}|{{ c }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "{1: 'a', (2, 3): 'b'}|{1: 'x', (2, 3): 'b'}" + }, + { + "id": "methods.dict_setdefault_existing_float", + "template": "{% set d = {1: 'a'} %}{{ d.setdefault(1.0, 'z') }}{{ d }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "a{1: 'a'}" + }, + { + "id": "methods.dict_update_unhashable_unchanged", + "template": "{% set d = {1: 'a'} %}{% do d.update([(2, 'b'), ([3], 'c')]) %}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "error": { + "phase": "render", + "type": "TypeError", + "message": "unhashable type: 'list'" + } + }, + { + "id": "methods.dict_store_item_non_string", + "template": "{% set d = {1: 'a'} %}{% do d.update({2: 'b'}) %}{{ d[2] }}{{ d.get(2) }}{{ d|length }}", + "context": { + "s": "Hello World", + "l": [ + 3, + 1, + 2 + ], + "d": { + "a": 1, + "b": 2 + }, + "csv": "a,b,,c", + "pad": " x " + }, + "env": { + "extensions": [ + "do" + ] + }, + "templates": {}, + "output": "bb2" } ] } diff --git a/test/parity/expected/operators.json b/test/parity/expected/operators.json index 17c396291..d6b718a69 100644 --- a/test/parity/expected/operators.json +++ b/test/parity/expected/operators.json @@ -2011,7 +2011,7 @@ "error": { "phase": "render", "type": "ZeroDivisionError", - "message": "float modulo" + "message": "float modulo by zero" } }, { @@ -2653,6 +2653,72 @@ "type": "TemplateSyntaxError", "message": "expected token 'end of print statement', got 'z'" } + }, + { + "id": "operators.in_dict_non_string_key", + "template": "{{ 'T' if 1 in {1: 'a'} else 'F' }}|{{ 'T' if '1' in {1: 'a'} else 'F' }}|{{ 'T' if 1.0 in {1: 'a'} else 'F' }}|{{ 'T' if (1, 2) in {(1, 2): 'a'} else 'F' }}|{{ 'T' if none in {none: 'a'} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "T|F|T|T|T" + }, + { + "id": "operators.eq_dict_non_string_keys", + "template": "{{ 'T' if {1: 'a'} == {1.0: 'a'} else 'F' }}|{{ 'T' if {1: 'a'} == {'1': 'a'} else 'F' }}|{{ 'T' if {(1, 2): 3} == {(1.0, 2): 3} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "T|F|T" + }, + { + "id": "operators.not_in_dict_non_string_key", + "template": "{{ 'T' if 2 not in {1: 'a'} else 'F' }}|{{ 'T' if True in {1: 'a'} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "T|T" } ] } diff --git a/test/parity/expected/statements.json b/test/parity/expected/statements.json index 12b886036..38cfd7b21 100644 --- a/test/parity/expected/statements.json +++ b/test/parity/expected/statements.json @@ -16040,6 +16040,86 @@ "env": {}, "templates": {}, "output": "21" + }, + { + "id": "statements.for_dict_int_keys", + "template": "{% for k in {1: 'a', 2: 'b'} %}{{ k + 1 }}{% endfor %}|{% for k, v in {2: 'a'}.items() %}{{ k }}{{ v }}{% endfor %}|{% for k, v in {20: 'x', 10: 'y'}|dictsort %}{{ k }}{% endfor %}", + "context": { + "l": [ + 3, + 1, + 2 + ], + "e": [], + "d": { + "a": 1, + "b": 2 + }, + "x": 3, + "tree": [ + { + "n": "a", + "c": [ + { + "n": "b", + "c": [] + }, + { + "n": "c", + "c": [ + { + "n": "d", + "c": [] + } + ] + } + ] + } + ] + }, + "env": {}, + "templates": {}, + "output": "23|2a|1020" + }, + { + "id": "statements.for_dict_tuple_key_unpack", + "template": "{% for k, v in {(1, 2): 'a'}.items() %}{{ k[0] }}{{ k[1] }}{{ v }}{% endfor %}", + "context": { + "l": [ + 3, + 1, + 2 + ], + "e": [], + "d": { + "a": 1, + "b": 2 + }, + "x": 3, + "tree": [ + { + "n": "a", + "c": [ + { + "n": "b", + "c": [] + }, + { + "n": "c", + "c": [ + { + "n": "d", + "c": [] + } + ] + } + ] + } + ] + }, + "env": {}, + "templates": {}, + "output": "12a" } ] } diff --git a/test/parity/expected/subscripts.json b/test/parity/expected/subscripts.json index 56fa4276f..6ec8e94b7 100644 --- a/test/parity/expected/subscripts.json +++ b/test/parity/expected/subscripts.json @@ -1239,6 +1239,102 @@ "env": {}, "templates": {}, "output": "[][]" + }, + { + "id": "subscripts.dict_unhashable_subscript", + "template": "[{{ {1:2}[[1]] }}]", + "context": { + "l": [ + 3, + 1, + 2, + 5 + ], + "s": "Hello World", + "d": { + "a": 1, + "b": { + "c": 2 + }, + "key with space": 3, + "items": "own" + }, + "users": [ + { + "name": "bob" + }, + { + "name": "alice" + } + ] + }, + "env": {}, + "templates": {}, + "output": "[]" + }, + { + "id": "subscripts.dict_subscript_float_and_bool", + "template": "{{ {1: 'a'}[1.0] }}|{{ {1: 'a'}[true] }}|{{ {1.5: 'f'}[1.5] }}", + "context": { + "l": [ + 3, + 1, + 2, + 5 + ], + "s": "Hello World", + "d": { + "a": 1, + "b": { + "c": 2 + }, + "key with space": 3, + "items": "own" + }, + "users": [ + { + "name": "bob" + }, + { + "name": "alice" + } + ] + }, + "env": {}, + "templates": {}, + "output": "a|a|f" + }, + { + "id": "subscripts.dict_subscript_missing_non_string", + "template": "[{{ {1: 'a'}[2] }}][{{ {1: 'a'}[none] }}]", + "context": { + "l": [ + 3, + 1, + 2, + 5 + ], + "s": "Hello World", + "d": { + "a": 1, + "b": { + "c": 2 + }, + "key with space": 3, + "items": "own" + }, + "users": [ + { + "name": "bob" + }, + { + "name": "alice" + } + ] + }, + "env": {}, + "templates": {}, + "output": "[][]" } ] } From 1f80a009d8941129ff05b5c3a685a1f561257c58 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 11:59:06 +0000 Subject: [PATCH 03/18] Lend host map keys and slim dictsort's sort items (0036) Reading a host map's entries copied every key into an owned value, and dictsort's sort items grew by a value; Render/dict_ops was +5.9%. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- src/filters.cpp | 43 ++++++++++++++---------------------------- src/internal_value.cpp | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/filters.cpp b/src/filters.cpp index 601a663ab..91c4f591e 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -468,10 +468,8 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont struct SortItem { KeyValuePair* pair; - // The key as UTF-8 text (lowered unless case-sensitive) when every key is a string + // The lowered key when case-insensitive and every key is a string std::string sortKey; - // The key otherwise; Python lowers only a str key, not the elements of a tuple - InternalValue sortValue; }; bool (*comparator)(const SortItem& left, const SortItem& right) = nullptr; bool lowerKeys = false; @@ -484,15 +482,22 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont { sortByKey = true; lowerKeys = !ConvertToBool(isCsVal); - keysAreStrings = std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); }); - if (keysAreStrings) + // Narrow text, as map keys are UTF-8 (a template dict converts wide keys) + keysAreStrings = std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return AsStringView(entry.key).has_value(); }); + if (keysAreStrings && lowerKeys) { comparator = [](const SortItem& left, const SortItem& right) { return left.sortKey < right.sortKey; }; } + else if (keysAreStrings) + { + comparator = [](const SortItem& left, const SortItem& right) { return *AsStringView(left.pair->key) < *AsStringView(right.pair->key); }; + } else { + // Python lowers only str keys, and a str key next to a key of another type + // fails the comparison either way, so other keys compare as they are comparator = [](const SortItem& left, const SortItem& right) { - return CompareForOrder(left.sortValue, right.sortValue, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive); + return CompareForOrder(left.pair->key, right.pair->key, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive); }; } } @@ -521,30 +526,10 @@ InternalValue DictSort::Filter(const InternalValue& baseVal, RenderContext& cont tempVector.reserve(entries.size()); for (auto& entry : entries) { - SortItem item{ &entry, std::string(), InternalValue() }; - if (sortByKey && keysAreStrings) - { - const auto text = AsString(entry.key); - if (lowerKeys) - { - detail::LowerUtf8(text, item.sortKey); - } - else - { - item.sortKey = text; - } - } - else if (sortByKey) + SortItem item{ &entry, std::string() }; + if (sortByKey && keysAreStrings && lowerKeys) { - if (lowerKeys && IsStringValue(entry.key)) - { - detail::LowerUtf8(AsString(entry.key), item.sortKey); - item.sortValue = InternalValue(item.sortKey); - } - else - { - item.sortValue = entry.key; - } + detail::LowerUtf8(*AsStringView(entry.key), item.sortKey); } tempVector.push_back(std::move(item)); } diff --git a/src/internal_value.cpp b/src/internal_value.cpp index 8d72cf6d3..c2a2d06bb 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -2022,6 +2022,21 @@ class GenericMapAdapter : public MapAccessorImpl> Holder m_values; }; +// A key of user data as LendItem lends its string items: a view while the data outlives the +// render, so reading entries copies no key text (Render/dict_ops) +template class Holder, typename T> +InternalValue LendKey([[maybe_unused]] const Holder& holder, const std::string& key) +{ + if constexpr (std::is_same_v, ByRef>) + { + return InternalValue(TargetStringView(std::string_view(key))); + } + else + { + return InternalValue(key); + } +} + template class Holder> class ValuesMapAdapter : public MapAccessorImpl> { @@ -2064,7 +2079,7 @@ class ValuesMapAdapter : public MapAccessorImpl> for (const auto& [key, value] : m_values.Get()) { - result.push_back(KeyValuePair{ InternalValue(key), LendItem(m_values, value) }); + result.push_back(KeyValuePair{ LendKey(m_values, key), LendItem(m_values, value) }); } return result; From fadd45c91727811337b7f8dc3e3d903b5a48ce0c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 14:23:29 +0000 Subject: [PATCH 04/18] Clear clang-tidy hits on the 0036 changes Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- src/global_functions.cpp | 42 ++++++++++++++++++++---------------- src/internal_value.cpp | 2 +- src/serialize_filters.cpp | 2 +- src/template_dict.cpp | 7 ++++-- src/template_dict.h | 1 + test/internal_value_test.cpp | 7 ++++-- 6 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/global_functions.cpp b/src/global_functions.cpp index d5a7eaa6b..47a6646bc 100644 --- a/src/global_functions.cpp +++ b/src/global_functions.cpp @@ -127,6 +127,29 @@ InternalValue CallRange(const CallParams& params, RenderContext&) return ListAdapter::CreateRange(start, stop, step); } +// One item of dict(pairs) into `result` +void CollectDictPair(const InternalValue& item, TemplateDict& result) +{ + if (const auto* pair = GetIf(&item)) + { + result.Set(pair->key, pair->value); + return; + } + // Any iterable of two items is a pair, a two-character string included + bool isConverted = false; + auto itemList = ConvertToList(item, isConverted, false); + if (!isConverted) + { + throw std::runtime_error("cannot convert dictionary update sequence element to a sequence"); + } + auto pair = itemList.ToValueList(); + if (pair.size() != 2) + { + throw std::runtime_error("dictionary update sequence element has wrong length; 2 is required"); + } + result.Set(pair[0], pair[1]); +} + // The items of dict(mapping_or_pairs, **kwargs), into `result` void CollectDictItems(const CallParams& params, const char* fnName, TemplateDict& result) { @@ -155,24 +178,7 @@ void CollectDictItems(const CallParams& params, const char* fnName, TemplateDict { for (const auto& item : *list) { - if (const auto* pair = GetIf(&item)) - { - result.Set(pair->key, pair->value); - continue; - } - // Any iterable of two items is a pair, a two-character string included - bool isConverted = false; - auto itemList = ConvertToList(item, isConverted, false); - if (!isConverted) - { - throw std::runtime_error("cannot convert dictionary update sequence element to a sequence"); - } - auto pair = itemList.ToValueList(); - if (pair.size() != 2) - { - throw std::runtime_error("dictionary update sequence element has wrong length; 2 is required"); - } - result.Set(pair[0], pair[1]); + CollectDictPair(item, result); } } else if (!IsEmpty(source)) diff --git a/src/internal_value.cpp b/src/internal_value.cpp index c2a2d06bb..4bdcfb21d 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -1867,7 +1867,7 @@ class SharedDictAdapter : public MapAccessorImpl [[nodiscard]] bool ShouldExtendLifetime() const override { return m_values.ShouldExtendLifetime(); } [[nodiscard]] bool IsEqual(const IComparable& other) const override { - auto* val = dynamic_cast(&other); + const auto* val = dynamic_cast(&other); if (!val) { return false; diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index 535166c6c..966ac80b3 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -58,7 +58,7 @@ struct PrettyPrinter : visitors::BaseVisitor { return ConvertToBool(Apply2(left, right, BinaryExpression::LogicalLt, BinaryExpression::CaseSensitive)); } - catch (const std::runtime_error&) + catch (const std::runtime_error&) // NOLINT(bugprone-empty-catch): the fallback below is the handling { // Tuples with items Python cannot order: ordered by type like the rest } diff --git a/src/template_dict.cpp b/src/template_dict.cpp index 8c118334b..97e9f9e49 100644 --- a/src/template_dict.cpp +++ b/src/template_dict.cpp @@ -1,19 +1,22 @@ #include "template_dict.h" +#include "internal_value.h" #include "value_visitors.h" #include +#include #include #include #include #include #include +#include +#include #include #include #include #include -#include namespace jinja2 { @@ -212,7 +215,7 @@ struct KeyTokenBuilder private: // A tuple item: the length of its token, then the token - const char* AddElement(const InternalValue& item) const + [[nodiscard]] const char* AddElement(const InternalValue& item) const { const auto lengthAt = m_out.size(); AppendBytes(m_out, uint32_t{ 0 }); diff --git a/src/template_dict.h b/src/template_dict.h index 6d9851e81..ee0b42cf9 100644 --- a/src/template_dict.h +++ b/src/template_dict.h @@ -3,6 +3,7 @@ #include "internal_value.h" +#include #include #include #include diff --git a/test/internal_value_test.cpp b/test/internal_value_test.cpp index d939cae64..27aaac253 100644 --- a/test/internal_value_test.cpp +++ b/test/internal_value_test.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -255,9 +256,11 @@ TEST(TemplateDictTest, KeyTokensFollowPythonEquality) EXPECT_EQ(token(tuple({ tuple({ one }), InternalValue(std::string("a")) })), token(tuple({ tuple({ InternalValue(1.0) }), InternalValue(std::wstring(L"a")) }))); EXPECT_EQ(token(InternalValue(std::string("a"))), token(InternalValue(std::wstring(L"a")))); - EXPECT_EQ(std::string("a"), *token(InternalValue(std::string("a")))); + EXPECT_EQ(std::optional("a"), token(InternalValue(std::string("a")))); // A string that spells the token of the integer 1 is another key - EXPECT_NE(token(one), token(InternalValue(*token(one)))); + const auto oneToken = token(one); + ASSERT_TRUE(oneToken.has_value()); + EXPECT_NE(oneToken, token(InternalValue(oneToken.value_or(std::string())))); EXPECT_NE(token(InternalValue(NoneValue{})), token(InternalValue(std::string("N")))); const auto range = [](int64_t start, int64_t stop, int64_t step) { return InternalValue(ListAdapter::CreateRange(start, stop, step)); }; From 530d07bcc33f1deb8c6010b925e54b8421188733 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 14:39:42 +0000 Subject: [PATCH 05/18] Guard tuple key hashing against deep nesting; object keys find nothing (0036) Verifier round 1: hashing a key recursed through nested tuples without a stack check, so a deeply nested tuple crashed where Python raises RecursionError; and objects, which have no key token yet, matched the empty-string key in lookups. The dict methods' cycle check also walks non-string keys now. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- docs/tasks/0036-non-string-mapping-keys.md | 5 ++- src/template_dict.cpp | 36 ++++++++++++++++------ src/template_dict.h | 8 +++-- src/value_methods.cpp | 6 ++-- test/parity/cases/operators.py | 1 + test/parity/expected/operators.json | 22 +++++++++++++ test/recursion_limits_test.cpp | 14 +++++++++ 7 files changed, 76 insertions(+), 16 deletions(-) diff --git a/docs/tasks/0036-non-string-mapping-keys.md b/docs/tasks/0036-non-string-mapping-keys.md index 7bf3e937d..08dfad908 100644 --- a/docs/tasks/0036-non-string-mapping-keys.md +++ b/docs/tasks/0036-non-string-mapping-keys.md @@ -39,6 +39,9 @@ Python's `{str(k): v for k, v in d.items()}` has it. A dict of string keys stays Changing this rule later is a break for callers who rely on it. *Not done:* object keys (`{namespace(): 1}`, host objects, callables, undefined) have no -identity token yet and all read as the empty string, as before; that is the first bullet of +identity token yet and all read as the empty string when stored, as before, while a lookup by +one finds nothing (`nothing in {'': 1}` is False); that is the first bullet of 0212 (`globals.object_dict_key` stays listed). All NaN keys are one key (Python compares NaN by identity). +A cycle through a key (`{% set d = {(ns, 1): 1} %}{% set ns.d = d %}`) leaks like the one +through a value: that is 0110's (the dict methods' cycle check walks the keys too). diff --git a/src/template_dict.cpp b/src/template_dict.cpp index 97e9f9e49..c0321a37d 100644 --- a/src/template_dict.cpp +++ b/src/template_dict.cpp @@ -1,6 +1,7 @@ #include "template_dict.h" #include "internal_value.h" +#include "recursion_guard.h" #include "value_visitors.h" #include @@ -38,6 +39,8 @@ struct UnhashableTypeGetter : visitors::BaseVisitor const char* operator()(const ListRef& list) const { + // A tuple can nest deeper than the stack: Python raises RecursionError + CheckStack(); if (list.GetRangeInfo()) { return nullptr; @@ -96,8 +99,9 @@ uint64_t RangeLength(const RangeInfo& range) // Appends the token of a value, and returns the type that made it unhashable instead struct KeyTokenBuilder { - explicit KeyTokenBuilder(std::string& out) + KeyTokenBuilder(std::string& out, bool& hasObject) : m_out(out) + , m_hasObject(hasObject) { } @@ -148,6 +152,7 @@ struct KeyTokenBuilder } const char* operator()(const ListRef& list) const { + CheckStack(); if (const auto* range = list.GetRangeInfo()) { // Equal ranges are equal keys: an empty range is one, and a single item has no step @@ -196,6 +201,7 @@ struct KeyTokenBuilder // A dictsort pair is the tuple (key, value) const char* operator()(const KeyValuePair& pair) const { + CheckStack(); m_out.push_back(DictKeyMark); m_out.push_back(KindTuple); AppendBytes(m_out, uint32_t{ 2 }); @@ -220,7 +226,7 @@ struct KeyTokenBuilder const auto lengthAt = m_out.size(); AppendBytes(m_out, uint32_t{ 0 }); const auto start = m_out.size(); - if (const char* type = Apply(item, m_out)) + if (const char* type = Apply(item, m_out, m_hasObject)) { return type; } @@ -230,8 +236,9 @@ struct KeyTokenBuilder } // Objects, undefined values and callables have no key of their own yet (docs/tasks/0212): - // they add nothing, so as a key each stands for the empty string, as dict keys did before - static void AddObject() {} + // they add nothing, so as a key each stands for the empty string, as dict keys did before, + // and a lookup by one finds nothing (MakeDictKey's hasObject) + void AddObject() const { m_hasObject = true; } template void AddString(std::basic_string_view str) const @@ -254,6 +261,7 @@ struct KeyTokenBuilder } std::string& m_out; + bool& m_hasObject; }; std::string EscapeString(const std::string& name) @@ -283,10 +291,16 @@ const char* UnhashableType(const InternalValue& value) return Apply(value); } -std::optional MakeDictKey(const InternalValue& key, const char** unhashableType) +std::optional MakeDictKey(const InternalValue& key, const char** unhashableType, bool* hasObject) { std::string token; - if (const char* type = Apply(key, token)) + bool objectFound = false; + const char* type = Apply(key, token, objectFound); + if (hasObject) + { + *hasObject = objectFound; + } + if (type) { if (unhashableType) { @@ -352,14 +366,16 @@ TemplateDict::iterator TemplateDict::FindString(const std::string& name) TemplateDict::const_iterator TemplateDict::Find(const InternalValue& key) const { - auto token = MakeDictKey(key); - return token ? m_items.find(*token) : m_items.end(); + bool hasObject = false; + auto token = MakeDictKey(key, nullptr, &hasObject); + return token && !hasObject ? m_items.find(*token) : m_items.end(); } TemplateDict::iterator TemplateDict::Find(const InternalValue& key) { - auto token = MakeDictKey(key); - return token ? m_items.find(*token) : m_items.end(); + bool hasObject = false; + auto token = MakeDictKey(key, nullptr, &hasObject); + return token && !hasObject ? m_items.find(*token) : m_items.end(); } TemplateDict::iterator TemplateDict::Set(const InternalValue& key, InternalValue value) diff --git a/src/template_dict.h b/src/template_dict.h index ee0b42cf9..0e6b3eb66 100644 --- a/src/template_dict.h +++ b/src/template_dict.h @@ -37,8 +37,9 @@ const char* UnhashableType(const InternalValue& value); // The token of a key, which is equal for the keys Python treats as one. Nullopt when the key // cannot be hashed, with the type to report in `unhashableType`: Python's TypeError // `unhashable type: 'list'`. Objects (namespace(), cycler(), host values), undefined values and -// callables have no token of their own yet (docs/tasks/0212): they read as the empty string -std::optional MakeDictKey(const InternalValue& key, const char** unhashableType = nullptr); +// callables have no token of their own yet (docs/tasks/0212): they read as the empty string, and +// `hasObject` tells a lookup that the token stands for no key of its own +std::optional MakeDictKey(const InternalValue& key, const char** unhashableType = nullptr, bool* hasObject = nullptr); // Throws Python's TypeError for a key MakeDictKey rejects [[noreturn]] void ThrowUnhashable(const char* type); @@ -75,7 +76,8 @@ class TemplateDict // A string key; end() when absent [[nodiscard]] const_iterator FindString(const std::string& name) const; [[nodiscard]] iterator FindString(const std::string& name); - // Any key; end() when absent or unhashable (the caller checks UnhashableType where Python raises) + // Any key; end() when absent or unhashable (the caller checks UnhashableType where Python + // raises), and for a key that holds an object, which has no token of its own yet (0212) [[nodiscard]] const_iterator Find(const InternalValue& key) const; [[nodiscard]] iterator Find(const InternalValue& key); diff --git a/src/value_methods.cpp b/src/value_methods.cpp index 04da2c7a6..16b61ae8c 100644 --- a/src/value_methods.cpp +++ b/src/value_methods.cpp @@ -1688,9 +1688,11 @@ bool Reaches(const InternalValue& val, const void* target, std::unordered_setGetMutableItems()) + const auto& dict = *map->GetMutableItems(); + for (const auto& [key, value] : dict) { - if (Reaches(value, target, visited)) + // A key that is not a string keeps its first spelling, which can hold the target too + if (Reaches(value, target, visited) || (dict.HasKeyValues() && Reaches(dict.KeyOf(key), target, visited))) { return true; } diff --git a/test/parity/cases/operators.py b/test/parity/cases/operators.py index 1deaa05e8..157879026 100644 --- a/test/parity/cases/operators.py +++ b/test/parity/cases/operators.py @@ -125,5 +125,6 @@ ("not_after_binary_operator", "{{ x + not z }}"), ('in_dict_non_string_key', "{{ 'T' if 1 in {1: 'a'} else 'F' }}|{{ 'T' if '1' in {1: 'a'} else 'F' }}|{{ 'T' if 1.0 in {1: 'a'} else 'F' }}|{{ 'T' if (1, 2) in {(1, 2): 'a'} else 'F' }}|{{ 'T' if none in {none: 'a'} else 'F' }}"), ('eq_dict_non_string_keys', "{{ 'T' if {1: 'a'} == {1.0: 'a'} else 'F' }}|{{ 'T' if {1: 'a'} == {'1': 'a'} else 'F' }}|{{ 'T' if {(1, 2): 3} == {(1.0, 2): 3} else 'F' }}"), + ('in_dict_object_key', "{{ 'T' if nothing in {'': 1} else 'F' }}|{{ 'T' if namespace() in {'': 1} else 'F' }}"), ('not_in_dict_non_string_key', "{{ 'T' if 2 not in {1: 'a'} else 'F' }}|{{ 'T' if True in {1: 'a'} else 'F' }}"), ] diff --git a/test/parity/expected/operators.json b/test/parity/expected/operators.json index d6b718a69..b1878e9dc 100644 --- a/test/parity/expected/operators.json +++ b/test/parity/expected/operators.json @@ -2698,6 +2698,28 @@ "templates": {}, "output": "T|F|T" }, + { + "id": "operators.in_dict_object_key", + "template": "{{ 'T' if nothing in {'': 1} else 'F' }}|{{ 'T' if namespace() in {'': 1} else 'F' }}", + "context": { + "x": 3, + "y": 2.5, + "l": [ + 3, + 1, + 2 + ], + "s": "Hello", + "d": { + "a": 1 + }, + "n": null, + "z": 0 + }, + "env": {}, + "templates": {}, + "output": "F|F" + }, { "id": "operators.not_in_dict_non_string_key", "template": "{{ 'T' if 2 not in {1: 'a'} else 'F' }}|{{ 'T' if True in {1: 'a'} else 'F' }}", diff --git a/test/recursion_limits_test.cpp b/test/recursion_limits_test.cpp index 96be89722..d32acfc7f 100644 --- a/test/recursion_limits_test.cpp +++ b/test/recursion_limits_test.cpp @@ -113,6 +113,20 @@ TEST_F(RecursionLimitsTest, ErrorMessage) EXPECT_NE(std::string::npos, ErrorToString(result.error()).find("error: Maximum recursion depth exceeded\n")) << ErrorToString(result.error()); } +TEST_F(RecursionLimitsTest, DeepTupleKeyIsRenderError) +{ + // A dict key is hashed through its nested tuples, as deep as the template made them + const std::string build = "{% set ns = namespace(t=(1,)) %}{% for i in range(20000) %}{% set ns.t = (ns.t,) %}{% endfor %}"; + for (const std::string use : { "{{ {ns.t: 1}|length }}", "{{ ns.t in {1: 1} }}", "{{ {1: 1}.get(ns.t) }}" }) + { + Template tpl(&m_env); + ASSERT_TRUE(tpl.Load(build + use).has_value()) << use; + auto result = tpl.RenderAsString({}); + ASSERT_FALSE(result.has_value()) << use; + EXPECT_EQ(ErrorCode::RecursionLimitExceeded, result.error().GetCode()) << use; + } +} + TEST_F(RecursionLimitsTest, InfiniteRecursionIsRenderError) { AddFile("include_self", "{% include 'include_self' %}"); From 944cb8061845dd9df59cd1ea9ee96641a675c4b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 19:20:21 +0000 Subject: [PATCH 06/18] Deep tuple key test: an optimised build may fit the depth on its stack CI's gcc-12 Release build hashed the 20000-deep key without reaching the stack limit, so the test now accepts Python's result or the recursion error, and still catches a crash. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- test/recursion_limits_test.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/test/recursion_limits_test.cpp b/test/recursion_limits_test.cpp index d32acfc7f..338223694 100644 --- a/test/recursion_limits_test.cpp +++ b/test/recursion_limits_test.cpp @@ -113,17 +113,29 @@ TEST_F(RecursionLimitsTest, ErrorMessage) EXPECT_NE(std::string::npos, ErrorToString(result.error()).find("error: Maximum recursion depth exceeded\n")) << ErrorToString(result.error()); } -TEST_F(RecursionLimitsTest, DeepTupleKeyIsRenderError) +TEST_F(RecursionLimitsTest, DeepTupleKeyRendersOrIsRenderError) { - // A dict key is hashed through its nested tuples, as deep as the template made them + // A dict key is hashed through its nested tuples, as deep as the template made them. The + // stack an optimised build needs for that depth may fit: then the key works as in Python const std::string build = "{% set ns = namespace(t=(1,)) %}{% for i in range(20000) %}{% set ns.t = (ns.t,) %}{% endfor %}"; - for (const std::string use : { "{{ {ns.t: 1}|length }}", "{{ ns.t in {1: 1} }}", "{{ {1: 1}.get(ns.t) }}" }) + const std::pair cases[] = { + { "{{ {ns.t: 1}|length }}", "1" }, + { "{{ ns.t in {1: 1} }}", "False" }, + { "{{ {1: 1}.get(ns.t) }}", "None" }, + }; + for (const auto& [use, expected] : cases) { Template tpl(&m_env); ASSERT_TRUE(tpl.Load(build + use).has_value()) << use; auto result = tpl.RenderAsString({}); - ASSERT_FALSE(result.has_value()) << use; - EXPECT_EQ(ErrorCode::RecursionLimitExceeded, result.error().GetCode()) << use; + if (result.has_value()) + { + EXPECT_EQ(expected, result.value()) << use; + } + else + { + EXPECT_EQ(ErrorCode::RecursionLimitExceeded, result.error().GetCode()) << use; + } } } From 8414a579bc620a780eaa165cb2af182a6b7835c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 19:22:13 +0000 Subject: [PATCH 07/18] Parity expectations: keep Python 3.12's ZeroDivisionError text The regeneration ran on Python 3.13, which words float modulo by zero differently from the pinned CI interpreter. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- test/parity/expected/operators.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parity/expected/operators.json b/test/parity/expected/operators.json index b1878e9dc..110db4e2b 100644 --- a/test/parity/expected/operators.json +++ b/test/parity/expected/operators.json @@ -2011,7 +2011,7 @@ "error": { "phase": "render", "type": "ZeroDivisionError", - "message": "float modulo by zero" + "message": "float modulo" } }, { From c8f4b06bc8259d26ac64b3b8946823e1ea85d348 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 18 Oct 2026 21:14:25 +0000 Subject: [PATCH 08/18] Deep tuple key test: stay within TSan's shadow call stack Under TSan the 20000-level key walk overran the runtime's fixed-size shadow call stack (from about 16000 levels, before the real stack ran low), so __tsan_func_entry wrote past it and the run crashed inside TSan. TSan builds now nest 10000 levels; ASan and plain builds keep the full depth. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- test/recursion_limits_test.cpp | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/recursion_limits_test.cpp b/test/recursion_limits_test.cpp index 338223694..cc00e5776 100644 --- a/test/recursion_limits_test.cpp +++ b/test/recursion_limits_test.cpp @@ -17,6 +17,14 @@ #include #endif +#ifdef __SANITIZE_THREAD__ +#define JINJA2CPP_TEST_TSAN 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define JINJA2CPP_TEST_TSAN 1 +#endif +#endif + using namespace jinja2; namespace @@ -117,7 +125,15 @@ TEST_F(RecursionLimitsTest, DeepTupleKeyRendersOrIsRenderError) { // A dict key is hashed through its nested tuples, as deep as the template made them. The // stack an optimised build needs for that depth may fit: then the key works as in Python - const std::string build = "{% set ns = namespace(t=(1,)) %}{% for i in range(20000) %}{% set ns.t = (ns.t,) %}{% endfor %}"; +#ifdef JINJA2CPP_TEST_TSAN + // TSan records every call on a fixed-size shadow stack, which this walk overruns from + // about 16000 levels, before the real stack runs low: __tsan_func_entry then writes past + // it and the run crashes inside TSan. ASan and the plain builds take the full depth + constexpr char depth[] = "10000"; +#else + constexpr char depth[] = "20000"; +#endif + const std::string build = std::string("{% set ns = namespace(t=(1,)) %}{% for i in range(") + depth + ") %}{% set ns.t = (ns.t,) %}{% endfor %}"; const std::pair cases[] = { { "{{ {ns.t: 1}|length }}", "1" }, { "{{ ns.t in {1: 1} }}", "False" }, @@ -380,13 +396,6 @@ TEST_F(RecursionLimitsTest, SmallThreadStack) // ==) still recurse, but stop with RecursionLimitExceeded like Python's RecursionError. namespace { -#ifdef __SANITIZE_THREAD__ -#define JINJA2CPP_TEST_TSAN 1 -#elif defined(__has_feature) -#if __has_feature(thread_sanitizer) -#define JINJA2CPP_TEST_TSAN 1 -#endif -#endif #ifdef JINJA2CPP_TEST_TSAN // Deep enough to overflow the walks' thread under TSan too. Deeper, the access history of // the whole run makes LLVM 18's TSan reset its traces, which fails its own CHECK From c129875afbb490f394ec5eed83d7e5527ae1db32 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 00:05:11 +0000 Subject: [PATCH 09/18] Hold tuple dict keys to MaxSequenceSize A lazy tuple such as ('x',) * 2**40 reports items it never stores, and hashing it as a dict key walked every one: the unhashable check and the key token both bypassed the sequence cap. The check now counts the items of all nested tuples against MaxSequenceSize and the token checks its byte size per element, so such a key fails fast with 'too long'. Found by Codex review on #594. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UBh4wYzMbbA9rW6VkD5Lh7 --- src/template_dict.cpp | 28 +++++++++++++++++++++++----- test/recursion_limits_test.cpp | 3 +++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/template_dict.cpp b/src/template_dict.cpp index c0321a37d..c970b74b5 100644 --- a/src/template_dict.cpp +++ b/src/template_dict.cpp @@ -37,6 +37,11 @@ struct UnhashableTypeGetter : visitors::BaseVisitor { using visitors::BaseVisitor::operator(); + explicit UnhashableTypeGetter(uint64_t& items) + : m_items(items) + { + } + const char* operator()(const ListRef& list) const { // A tuple can nest deeper than the stack: Python raises RecursionError @@ -49,9 +54,13 @@ struct UnhashableTypeGetter : visitors::BaseVisitor { return "list"; } + // A lazy tuple (('x',) * 2**40) reports items it never stores: the walk is held to + // MaxSequenceSize items over all its tuples, like any sequence the template builds + m_items += list.GetSize().value_or(0); + CheckSequenceSize(m_items, "a dict key"); const char* result = nullptr; - list.ForEach([&result](const InternalValue& item) { - result = Apply(item); + list.ForEach([this, &result](const InternalValue& item) { + result = Apply(item, m_items); return result == nullptr; }); return result; @@ -60,7 +69,10 @@ struct UnhashableTypeGetter : visitors::BaseVisitor { return map.GetObjectClass() || map.GetAttrPolicy() == MapAttrPolicy::KeysFirst ? nullptr : "dict"; } - const char* operator()(const KeyValuePair& pair) const { return Apply(pair.value); } + const char* operator()(const KeyValuePair& pair) const { return Apply(pair.value, m_items); } + +private: + uint64_t& m_items; }; template @@ -179,11 +191,16 @@ struct KeyTokenBuilder { return "list"; } + // A lazy tuple (('x',) * 2**40) reports items it never stores; its token would, so it + // is held to MaxSequenceSize like any sequence the template builds + const auto size = list.GetSize().value_or(0); + CheckSequenceSize(size, "a dict key"); m_out.push_back(DictKeyMark); m_out.push_back(KindTuple); - AppendBytes(m_out, static_cast(list.GetSize().value_or(0))); + AppendBytes(m_out, static_cast(size)); const char* result = nullptr; list.ForEach([this, &result](const InternalValue& item) { + CheckSequenceSize(m_out.size(), "a dict key"); result = AddElement(item); return result == nullptr; }); @@ -288,7 +305,8 @@ bool IsSpelledToken(const std::string& stored) const char* UnhashableType(const InternalValue& value) { - return Apply(value); + uint64_t items = 0; + return Apply(value, items); } std::optional MakeDictKey(const InternalValue& key, const char** unhashableType, bool* hasObject) diff --git a/test/recursion_limits_test.cpp b/test/recursion_limits_test.cpp index cc00e5776..f30a5fef4 100644 --- a/test/recursion_limits_test.cpp +++ b/test/recursion_limits_test.cpp @@ -320,6 +320,9 @@ TEST_F(RecursionLimitsTest, HugeSequencesFailFast) "{{ ('a\n' * 1000)|indent(2**30) }}", "{{ range(2**40)|reverse|first }}", "{{ ['x']|tojson(2**40) }}", + "{{ {('x',) * 2**40: 1} }}", + "{{ {1: 1}.get(('x',) * 1000000000) }}", + "{{ ('x' * 2**25,) * 4 in {} }}", }; for (const auto& source : templates) { From c4a0aaf5cedbf06913d6c68229f0e490f8ed7de2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 00:44:17 +0000 Subject: [PATCH 10/18] Add scripts/abi_namespace.py: wrap namespace jinja2 in inline namespace v2 (0076) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Ua8ABM1TRsAvso2sWGHFSw --- scripts/abi_namespace.py | 214 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100755 scripts/abi_namespace.py diff --git a/scripts/abi_namespace.py b/scripts/abi_namespace.py new file mode 100755 index 000000000..9b6e3652b --- /dev/null +++ b/scripts/abi_namespace.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +"""Keep every `namespace jinja2` in include/ and src/ inside the inline ABI namespace (docs/tasks/0076). + + scripts/abi_namespace.py # rewrite the files in place + scripts/abi_namespace.py --check # list what would change, exit 1 if anything would + +Public and internal code live in `jinja2::v2`, an inline namespace: users still write +`jinja2::Value`, but the mangled names carry the major version, so a program built against +1.x headers fails to link with a 2.x library instead of misbehaving. The tag is spelled out +in every file rather than hidden behind a macro (house clang-format and clang-tidy handle the +literal form); this script writes it so that nobody has to: + + namespace jinja2 namespace jinja2 + { { + ... -> inline namespace v2 + } { + ... + } // namespace v2 + } + +A nested `namespace jinja2::detail` becomes `namespace jinja2 { inline namespace v2 { +namespace detail` (C++17 cannot nest an inline namespace in that form). An opening already +followed by `inline namespace vN` with another N gets the new tag, so the 3.0 retag is one +run with TAG changed. The vendored polymorphic_value headers are rewritten in their own +upstream brace style; robin_hood.h and lexertk.h do not use namespace jinja2. +""" + +import argparse +import os +import re +import subprocess +import sys + +TAG = "v2" + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROOTS = ("include", "src") +OPENING = re.compile(r"^namespace jinja2(?P(?:::\w+)*)(?P\s*\{)?\s*$") +INLINE = re.compile(r"^inline namespace (?Pv\d+)(?P\s*\{)?\s*$") +CLOSE_COMMENT = re.compile(r"^\}(?P\s*)//\s*namespace\s+(?P[\w:]+)\s*$") + + +def code_mask(text): + """Copy of text with comments, string and character literals blanked, newlines kept.""" + out = list(text) + i, n = 0, len(text) + + def blank(a, b): + for k in range(a, b): + if out[k] != "\n": + out[k] = " " + + while i < n: + c = text[i] + if text.startswith("//", i): + j = text.find("\n", i) + j = n if j < 0 else j + blank(i, j) + i = j + elif text.startswith("/*", i): + j = text.find("*/", i + 2) + j = n if j < 0 else j + 2 + blank(i, j) + i = j + elif c == '"' or c == "'": + # R"delim( ... )delim", with any encoding prefix + prefix = re.search(r"(?:u8|[uUL])?R$", text[max(0, i - 3) : i]) + before = i - len(prefix.group(0)) - 1 if prefix else -1 + if c == '"' and prefix and (before < 0 or not (text[before].isalnum() or text[before] == "_")): + m = re.match(r'"([^(\s]*)\(', text[i:]) + if m: + end = text.find(")" + m.group(1) + '"', i + len(m.group(0))) + end = n if end < 0 else end + len(m.group(1)) + 2 + blank(i + 1, end - 1) + i = end + continue + if c == "'" and i > 0 and text[i - 1].isalnum() and i + 1 < n and text[i + 1].isalnum(): + i += 1 # digit separator: 1'000 + continue + j = i + 1 + while j < n and text[j] != c and text[j] != "\n": + j += 2 if text[j] == "\\" else 1 + blank(i + 1, min(j, n)) + i = j + 1 + else: + i += 1 + return "".join(out) + + +def matching_close(mask_lines, line, col): + """Line index of the `}` closing the `{` at (line, col) of the masked lines.""" + depth = 0 + for li in range(line, len(mask_lines)): + start = col if li == line else 0 + for ch in mask_lines[li][start:]: + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return li + raise ValueError("unbalanced braces") + + +def brace_after(mask_lines, line): + """(line, col) of the first `{` at or after the end of `line` (the opening's brace).""" + for li in range(line, len(mask_lines)): + col = mask_lines[li].find("{") + if col >= 0: + return li, col + if li > line and mask_lines[li].strip(): + break + raise ValueError("namespace opening without a brace") + + +def next_code_line(mask_lines, line): + for li in range(line + 1, len(mask_lines)): + if mask_lines[li].strip(): + return li + return None + + +def rewrite(text): + """Return text with every namespace jinja2 opening wrapped in `inline namespace TAG`.""" + lines = text.split("\n") + # Each pass fixes the first opening that needs it; the next pass re-reads the result. + for _ in range(10000): + mask_lines = code_mask("\n".join(lines)).split("\n") + edit = None + for li, raw in enumerate(lines): + m = OPENING.match(mask_lines[li].rstrip()) + if not m: + continue + same_line = bool(m.group("brace")) + open_line, open_col = brace_after(mask_lines, li) + close_line = matching_close(mask_lines, open_line, open_col) + close_text = lines[close_line] + cm = CLOSE_COMMENT.match(close_text.strip()) + space = cm.group("space") if cm else " " + if m.group("nested"): + inner = m.group("nested")[2:] + head = ["namespace jinja2", "{", f"inline namespace {TAG}", "{", f"namespace {inner}"] + if same_line: + head = ["namespace jinja2 {", f"inline namespace {TAG} {{", f"namespace {inner} {{"] + closing = "}" + space + f"// namespace {inner}" if cm else close_text + edit = (li, li + 1, head, close_line, [closing, "}" + space + f"// namespace {TAG}", "}" + space + "// namespace jinja2"]) + break + nxt = next_code_line(mask_lines, open_line if not same_line else li) + im = INLINE.match(mask_lines[nxt].strip()) if nxt is not None else None + if im: + if im.group("tag") == TAG: + continue + inline_brace_line, inline_col = brace_after(mask_lines, nxt) + inline_close = matching_close(mask_lines, inline_brace_line, inline_col) + lines[nxt] = lines[nxt].replace(im.group("tag"), TAG, 1) + icm = CLOSE_COMMENT.match(lines[inline_close].strip()) + if icm: + lines[inline_close] = "}" + icm.group("space") + f"// namespace {TAG}" + edit = () + break + insert = [f"inline namespace {TAG} {{"] if same_line else [f"inline namespace {TAG}", "{"] + edit = (open_line + 1, open_line + 1, insert, close_line, ["}" + space + f"// namespace {TAG}", close_text]) + break + if edit is None: + return "\n".join(lines) + if edit == (): + continue + a, b, new_open, close_line, new_close = edit + lines[close_line:close_line + 1] = new_close + lines[a:b] = new_open + raise RuntimeError("too many namespace openings") + + +def files(): + out = subprocess.run(["git", "ls-files", "--", *ROOTS], cwd=REPO, capture_output=True, text=True, check=True).stdout + for path in out.split(): + if path.endswith((".h", ".cpp")) and not path.endswith(("robin_hood.h", "lexertk.h")): + yield path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--check", action="store_true", help="change nothing; exit 1 if a file would change") + args = parser.parse_args() + + changed = [] + for path in files(): + full = os.path.join(REPO, path) + with open(full, encoding="utf-8", newline="") as f: + text = f.read() + try: + new = rewrite(text) + except ValueError as e: + print(f"{path}: {e}", file=sys.stderr) + return 2 + if new != text: + changed.append(path) + if not args.check: + with open(full, "w", encoding="utf-8", newline="") as f: + f.write(new) + + if args.check: + for path in changed: + print(f"{path}: namespace jinja2 not wrapped in inline namespace {TAG}") + if changed: + print("Run scripts/abi_namespace.py and commit the result (docs/tasks/0076).") + return 1 + return 0 + print(f"{len(changed)} files rewritten") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From ae4649c98baaf1999e99096f7d39ef7a4dca2b8d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 00:44:18 +0000 Subject: [PATCH 11/18] Wrap include/ and src/ in inline namespace jinja2::v2 (0076) Pure output of scripts/abi_namespace.py; no hand edits. Listed in .git-blame-ignore-revs. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Ua8ABM1TRsAvso2sWGHFSw --- include/jinja2cpp/binding/boost_json.h | 3 +++ include/jinja2cpp/binding/nlohmann_json.h | 3 +++ include/jinja2cpp/binding/rapid_json.h | 3 +++ include/jinja2cpp/error_info.h | 3 +++ include/jinja2cpp/filesystem_handler.h | 3 +++ include/jinja2cpp/fwd.h | 3 +++ include/jinja2cpp/generic_list.h | 3 +++ include/jinja2cpp/generic_list_iterator.h | 10 ++++++++-- include/jinja2cpp/make_generic_list.h | 3 +++ include/jinja2cpp/markup.h | 3 +++ .../polymorphic_value/indirect_cxx14.h | 2 ++ .../polymorphic_value/polymorphic_cxx14.h | 8 ++++++++ include/jinja2cpp/reflected_value.h | 3 +++ include/jinja2cpp/string_helpers.h | 3 +++ include/jinja2cpp/template.h | 3 +++ include/jinja2cpp/template_env.h | 3 +++ include/jinja2cpp/user_callable.h | 3 +++ include/jinja2cpp/utils/i_comparable.h | 3 +++ include/jinja2cpp/utils/ordered_map.h | 3 +++ include/jinja2cpp/value.h | 3 +++ include/jinja2cpp/value_ptr.h | 20 +++++++++++++++---- src/binding/boost_json_parser.h | 3 +++ src/binding/nlohmann_json_parser.h | 3 +++ src/binding/rapid_json_parser.h | 3 +++ src/cycle_collector.cpp | 3 +++ src/cycle_collector.h | 3 +++ src/error_handling.h | 3 +++ src/error_info.cpp | 3 +++ src/expression_evaluator.cpp | 3 +++ src/expression_evaluator.h | 3 +++ src/expression_parser.cpp | 3 +++ src/expression_parser.h | 3 +++ src/filesystem_handler.cpp | 3 +++ src/filter_factories.cpp | 3 +++ src/filters.cpp | 10 ++++++++-- src/filters.h | 3 +++ src/function_base.h | 3 +++ src/generic_adapters.h | 3 +++ src/generic_list.cpp | 3 +++ src/global_functions.cpp | 3 +++ src/helpers.h | 3 +++ src/html_entities.h | 10 ++++++++-- src/internal_value.cpp | 3 +++ src/internal_value.h | 3 +++ src/lexer.cpp | 3 +++ src/lexer.h | 3 +++ src/load_settings.h | 10 ++++++++-- src/lookup_result.h | 3 +++ src/loop_attr.h | 3 +++ src/make_unexpected.h | 3 +++ src/markup.h | 3 +++ src/name_resolver.cpp | 3 +++ src/name_resolver.h | 3 +++ src/node_arena.cpp | 3 +++ src/node_arena.h | 3 +++ src/out_stream.cpp | 3 +++ src/out_stream.h | 3 +++ src/python_format.cpp | 3 +++ src/python_format.h | 3 +++ src/recursion_guard.cpp | 3 +++ src/recursion_guard.h | 3 +++ src/render_context.h | 3 +++ src/render_workspace.cpp | 3 +++ src/render_workspace.h | 3 +++ src/renderer.h | 3 +++ src/serialize_filters.cpp | 10 ++++++++-- src/slot_frame.h | 3 +++ src/statements.cpp | 3 +++ src/statements.h | 3 +++ src/string_builder.h | 3 +++ src/string_converter_filter.cpp | 20 +++++++++++++++---- src/template.cpp | 3 +++ src/template_env.cpp | 3 +++ src/template_env_impl.h | 10 ++++++++-- src/template_impl.h | 3 +++ src/template_parser.cpp | 3 +++ src/template_parser.h | 3 +++ src/template_slots.h | 3 +++ src/tester_factories.cpp | 3 +++ src/testers.cpp | 10 ++++++++-- src/testers.h | 3 +++ src/undefined.cpp | 3 +++ src/undefined.h | 3 +++ src/unicode_case.h | 10 ++++++++-- src/unicode_printable.h | 10 ++++++++-- src/unicode_tables.h | 3 +++ src/value.cpp | 3 +++ src/value_methods.cpp | 10 ++++++++-- src/value_methods.h | 3 +++ src/value_visitors.h | 3 +++ 90 files changed, 350 insertions(+), 28 deletions(-) diff --git a/include/jinja2cpp/binding/boost_json.h b/include/jinja2cpp/binding/boost_json.h index 3a8449279..a08a73e8c 100644 --- a/include/jinja2cpp/binding/boost_json.h +++ b/include/jinja2cpp/binding/boost_json.h @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -247,6 +249,7 @@ struct Reflector } }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_BINDING_BOOST_JSON_H diff --git a/include/jinja2cpp/binding/nlohmann_json.h b/include/jinja2cpp/binding/nlohmann_json.h index 6f0ebb80e..91883131c 100644 --- a/include/jinja2cpp/binding/nlohmann_json.h +++ b/include/jinja2cpp/binding/nlohmann_json.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -237,6 +239,7 @@ struct Reflector : detail::NLohmannJsonReflector> return GenericMap([accessor = detail::RapidJsonObjectAccessor>(val)]() { return &accessor; }); } }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_BINDING_RAPID_JSON_H diff --git a/include/jinja2cpp/error_info.h b/include/jinja2cpp/error_info.h index faa989d33..af5f4dc2e 100644 --- a/include/jinja2cpp/error_info.h +++ b/include/jinja2cpp/error_info.h @@ -14,6 +14,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ /*! * \brief Type of the error * @@ -197,6 +199,7 @@ using ErrorInfoTpl JINJA2CPP_DEPRECATED("jinja2cpp-2: BasicErrorInfo") = BasicEr JINJA2CPP_EXPORT std::ostream& operator<<(std::ostream& os, const ErrorInfo& res); JINJA2CPP_EXPORT std::wostream& operator<<(std::wostream& os, const ErrorInfoW& res); +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/filesystem_handler.h b/include/jinja2cpp/filesystem_handler.h index f5b0556c5..f89ffc02b 100644 --- a/include/jinja2cpp/filesystem_handler.h +++ b/include/jinja2cpp/filesystem_handler.h @@ -19,6 +19,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ template using FileStreamPtr = std::unique_ptr, void (*)(std::basic_istream*)>; @@ -207,6 +209,7 @@ class JINJA2CPP_EXPORT RealFileSystem : public IFilesystemHandler private: std::string m_rootFolder; }; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/fwd.h b/include/jinja2cpp/fwd.h index ae7a3cd75..46844153c 100644 --- a/include/jinja2cpp/fwd.h +++ b/include/jinja2cpp/fwd.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ class Value; class GenericList; class GenericMap; @@ -59,6 +61,7 @@ using ErrorInfo = BasicErrorInfo; using ErrorInfoW = BasicErrorInfo; using ValuesList = std::vector; using FilesystemHandlerPtr = std::shared_ptr; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_FWD_H diff --git a/include/jinja2cpp/generic_list.h b/include/jinja2cpp/generic_list.h index 86622e1ee..012624cbc 100644 --- a/include/jinja2cpp/generic_list.h +++ b/include/jinja2cpp/generic_list.h @@ -18,6 +18,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ /*! * \brief Interface for accessing items in list by the indexes * @@ -287,6 +289,7 @@ inline ListEnumeratorPtr IListItemAccessor::MakeEnumerator(Args&&... args) { return ListEnumeratorPtr{ types::in_place_type_t{}, std::forward(args)... }; } +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/generic_list_iterator.h b/include/jinja2cpp/generic_list_iterator.h index 45eab1f69..d5b017568 100644 --- a/include/jinja2cpp/generic_list_iterator.h +++ b/include/jinja2cpp/generic_list_iterator.h @@ -15,7 +15,11 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { class JINJA2CPP_EXPORT GenericListIterator { @@ -124,7 +128,9 @@ class JINJA2CPP_EXPORT GenericListIterator Value m_current; }; -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/make_generic_list.h b/include/jinja2cpp/make_generic_list.h index f1f50dc2e..17e45a473 100644 --- a/include/jinja2cpp/make_generic_list.h +++ b/include/jinja2cpp/make_generic_list.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ /*! * \brief Generator of a generated list, see \ref MakeGenericList(ListGenerator) * @@ -513,6 +515,7 @@ inline GenericList MakeGenericList(ListGenerator fn) return detail::MakeGeneratedList(std::move(fn)); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_MAKE_GENERIC_LIST_H diff --git a/include/jinja2cpp/markup.h b/include/jinja2cpp/markup.h index 7e956ad1f..19dae86db 100644 --- a/include/jinja2cpp/markup.h +++ b/include/jinja2cpp/markup.h @@ -8,6 +8,8 @@ namespace jinja2 { +inline namespace v2 +{ /*! * \brief Text that is already safe HTML: Python's `markupsafe.Markup` * @@ -115,6 +117,7 @@ class BasicMarkup using Markup = BasicMarkup; //! Safe wide text, see \ref BasicMarkup using WMarkup = BasicMarkup; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_MARKUP_H diff --git a/include/jinja2cpp/polymorphic_value/indirect_cxx14.h b/include/jinja2cpp/polymorphic_value/indirect_cxx14.h index 96efe3106..ad16920f1 100644 --- a/include/jinja2cpp/polymorphic_value/indirect_cxx14.h +++ b/include/jinja2cpp/polymorphic_value/indirect_cxx14.h @@ -36,6 +36,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // with it at link time. namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { #ifndef JINJA2CPP_XYZ_IN_PLACE_DEFINED @@ -616,6 +617,7 @@ indirect(std::allocator_arg_t, Alloc, Value) -> indirect< } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 template diff --git a/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h b/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h index b41d702a0..6ea3042d2 100644 --- a/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h +++ b/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h @@ -47,12 +47,14 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #define JINJA2CPP_XYZ_IN_PLACE_TYPE_DEFINED namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { template struct in_place_type_t {}; } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_IN_PLACE_TYPE_DEFINED @@ -60,6 +62,7 @@ struct in_place_type_t {}; #define JINJA2CPP_XYZ_UNREACHABLE_DEFINED namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { [[noreturn]] inline void unreachable() { // LCOV_EXCL_LINE @@ -73,6 +76,7 @@ namespace xyz { } } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_UNREACHABLE_DEFINED @@ -85,6 +89,7 @@ namespace xyz { // We duplicate implementations to allow this header to work as a single // include. https://godbolt.org needs single-file includes. namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { namespace detail { @@ -121,10 +126,12 @@ class empty_base_optimization : private T { } // namespace detail } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_EMPTY_BASE_DEFINED namespace jinja2 { +inline namespace v2 { namespace detail { namespace xyz { @@ -505,6 +512,7 @@ class polymorphic : private detail::empty_base_optimization { } // namespace xyz } // namespace detail +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_XYZ_POLYMORPHIC_H_ diff --git a/include/jinja2cpp/reflected_value.h b/include/jinja2cpp/reflected_value.h index 18a8c9e95..84bd21f6d 100644 --- a/include/jinja2cpp/reflected_value.h +++ b/include/jinja2cpp/reflected_value.h @@ -23,6 +23,8 @@ namespace jinja2 { +inline namespace v2 +{ /*! * \brief Reflect the arbitrary C++ type to the Jinja2C++ engine @@ -687,6 +689,7 @@ Value Reflect(T&& val) return Reflector::Create(std::forward(val)); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_REFLECTED_VALUE_H diff --git a/include/jinja2cpp/string_helpers.h b/include/jinja2cpp/string_helpers.h index b8d27868c..f626f830d 100644 --- a/include/jinja2cpp/string_helpers.h +++ b/include/jinja2cpp/string_helpers.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { template @@ -374,6 +376,7 @@ inline std::string AsString(const Value& val) { return std::visit(detail::String //! The stored string, converted if needed, or an empty string for any other value, as in 1.x; \ref ToWString prints every value JINJA2CPP_DEPRECATED("jinja2cpp-2: ToWString") inline std::wstring AsWString(const Value& val) { return std::visit(detail::WStringGetter(), val.GetData()); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_STRING_HELPERS_H diff --git a/include/jinja2cpp/template.h b/include/jinja2cpp/template.h index 60c2cef2f..55e746638 100644 --- a/include/jinja2cpp/template.h +++ b/include/jinja2cpp/template.h @@ -27,6 +27,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ class ITemplateImpl; template class TemplateImpl; @@ -235,6 +237,7 @@ using Template = BasicTemplate; //! Template with wide (`wchar_t`) source and output using TemplateW = BasicTemplate; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/template_env.h b/include/jinja2cpp/template_env.h index 918923270..a787cbb56 100644 --- a/include/jinja2cpp/template_env.h +++ b/include/jinja2cpp/template_env.h @@ -22,6 +22,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -379,6 +381,7 @@ class JINJA2CPP_EXPORT TemplateEnv std::shared_ptr m_impl; }; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/user_callable.h b/include/jinja2cpp/user_callable.h index 78b152e4b..b98d87a38 100644 --- a/include/jinja2cpp/user_callable.h +++ b/include/jinja2cpp/user_callable.h @@ -17,6 +17,8 @@ namespace jinja2 { +inline namespace v2 +{ #ifndef JINJA2CPP_NO_DOXYGEN namespace detail { @@ -455,6 +457,7 @@ auto MakeCallable(Fn&& f) { return UserCallable{ [=, fn = std::forward(f)](const UserCallableParams&) { return fn(); }, {} }; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_USER_CALLABLE_H diff --git a/include/jinja2cpp/utils/i_comparable.h b/include/jinja2cpp/utils/i_comparable.h index 2f1cbc6de..0860ca1a2 100644 --- a/include/jinja2cpp/utils/i_comparable.h +++ b/include/jinja2cpp/utils/i_comparable.h @@ -8,6 +8,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ /*! * \brief Equality comparison for the extension interfaces (list and map accessors, enumerators, filesystem handlers) @@ -28,6 +30,7 @@ struct JINJA2CPP_EXPORT IComparable [[nodiscard]] virtual bool IsEqual(const IComparable& other) const { return this == &other; } }; +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/utils/ordered_map.h b/include/jinja2cpp/utils/ordered_map.h index 6e53b9418..3d120c447 100644 --- a/include/jinja2cpp/utils/ordered_map.h +++ b/include/jinja2cpp/utils/ordered_map.h @@ -17,6 +17,8 @@ namespace jinja2 { +inline namespace v2 +{ // A hash map that iterates in insertion order, like a Python dict (docs/tasks/0031, 0043). It is the base of // the public ValuesMap and of the library's own dicts. @@ -408,6 +410,7 @@ class OrderedMap std::unordered_map m_index; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_UTILS_ORDERED_MAP_H diff --git a/include/jinja2cpp/value.h b/include/jinja2cpp/value.h index 2a05e7ece..63a2d0573 100644 --- a/include/jinja2cpp/value.h +++ b/include/jinja2cpp/value.h @@ -30,6 +30,8 @@ JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_BEGIN namespace jinja2 { +inline namespace v2 +{ //! Python `None`: what a default-constructed \ref Value holds struct NoneValue { @@ -1356,6 +1358,7 @@ decltype(auto) Visit(Fn&& fn, Value& val) return detail::VisitUnwrapped(std::forward(fn), val.GetData()); } +} // namespace v2 } // namespace jinja2 JINJA2CPP_SUPPRESS_DLL_INTERFACE_WARNINGS_END diff --git a/include/jinja2cpp/value_ptr.h b/include/jinja2cpp/value_ptr.h index debdb2d43..6638a919b 100644 --- a/include/jinja2cpp/value_ptr.h +++ b/include/jinja2cpp/value_ptr.h @@ -4,7 +4,11 @@ #include "jinja2cpp/polymorphic_value/indirect_cxx14.h" #include "jinja2cpp/polymorphic_value/polymorphic_cxx14.h" -namespace jinja2::types +namespace jinja2 +{ +inline namespace v2 +{ +namespace types { // Only the names the library's headers use, not the whole vendored namespace @@ -20,9 +24,15 @@ auto MakeValuePtr(Ts&&... ts) return polymorphic(in_place_type_t{}, std::forward(ts)...); } -} // namespace jinja2::types +} // namespace types +} // namespace v2 +} // namespace jinja2 -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // Owns one heap-allocated T and copies it deeply, as C++26 std::indirect does; T may be // incomplete where the wrapper is declared. Holds the recursive alternatives of Value. @@ -30,6 +40,8 @@ template using indirect = xyz::indirect; // The tag that selects indirect's in-place constructor using in_place_t = xyz::in_place_t; -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_VALUE_PTR_H diff --git a/src/binding/boost_json_parser.h b/src/binding/boost_json_parser.h index d60b41a58..bb12a5606 100644 --- a/src/binding/boost_json_parser.h +++ b/src/binding/boost_json_parser.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ template nonstd::expected Parse(std::basic_string_view json, boost::anys::unique_any& metadataJson) @@ -29,6 +31,7 @@ nonstd::expected Parse(std::basic_string_view json, b return Reflect(value); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_BOOST_JSON_PARSER_H diff --git a/src/binding/nlohmann_json_parser.h b/src/binding/nlohmann_json_parser.h index 50e65c6f0..9e9d5f012 100644 --- a/src/binding/nlohmann_json_parser.h +++ b/src/binding/nlohmann_json_parser.h @@ -9,6 +9,8 @@ namespace jinja2 { +inline namespace v2 +{ template nonstd::expected Parse(std::basic_string_view json, boost::anys::unique_any& metadataJson) @@ -26,6 +28,7 @@ nonstd::expected Parse(std::basic_string_view json, b } } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_NLOHMANN_JSON_PARSER_H diff --git a/src/binding/rapid_json_parser.h b/src/binding/rapid_json_parser.h index b0ba13634..6ddcb94de 100644 --- a/src/binding/rapid_json_parser.h +++ b/src/binding/rapid_json_parser.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -68,6 +70,7 @@ nonstd::expected Parse(std::basic_string_view json, b return Reflect(jsonDoc); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RAPID_JSON_PARSER_H diff --git a/src/cycle_collector.cpp b/src/cycle_collector.cpp index 8dfb8df3a..f447f1e0b 100644 --- a/src/cycle_collector.cpp +++ b/src/cycle_collector.cpp @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -262,4 +264,5 @@ void CycleCollector::Collect() noexcept m_notes->candidates.clear(); } +} // namespace v2 } // namespace jinja2 diff --git a/src/cycle_collector.h b/src/cycle_collector.h index 05c4b9e24..99cbb274e 100644 --- a/src/cycle_collector.h +++ b/src/cycle_collector.h @@ -10,6 +10,8 @@ namespace jinja2 { +inline namespace v2 +{ // Frees the reference cycles among the lists, dicts and namespaces a render builds // (docs/tasks/0110). They are shared by reference count, as Python's are, so a container @@ -78,6 +80,7 @@ class CycleCollector std::unique_ptr m_notes; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_CYCLE_COLLECTOR_H diff --git a/src/error_handling.h b/src/error_handling.h index 726a67e43..8eafae953 100644 --- a/src/error_handling.h +++ b/src/error_handling.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ struct ParseError { @@ -49,5 +51,6 @@ inline auto MakeParseError(ErrorCode code, Token tok, std::initializer_list namespace jinja2 { +inline namespace v2 +{ template void RenderErrorInfo(std::basic_string& result, const BasicErrorInfo& errInfo) @@ -327,4 +329,5 @@ std::wostream& operator<<(std::wostream& os, const ErrorInfoW& res) os << res.ToString(); return os; } +} // namespace v2 } // namespace jinja2 diff --git a/src/expression_evaluator.cpp b/src/expression_evaluator.cpp index e17293535..a279d828e 100644 --- a/src/expression_evaluator.cpp +++ b/src/expression_evaluator.cpp @@ -41,6 +41,8 @@ using namespace std::string_literals; namespace jinja2 { +inline namespace v2 +{ // Out of line, so that the paths without autoescape stay small void WriteEscaped(OutStream& stream, const InternalValue& val, IRendererCallback* callback) @@ -1589,4 +1591,5 @@ ArenaCallParams ArenaCallParams::Make(NodeArena& nodes, const CallParamsInfo& pa result.kwParams = nodes.MakeSpan(kwParams); return result; } +} // namespace v2 } // namespace jinja2 diff --git a/src/expression_evaluator.h b/src/expression_evaluator.h index 06cf78ecb..dfa99c9a3 100644 --- a/src/expression_evaluator.h +++ b/src/expression_evaluator.h @@ -33,6 +33,8 @@ namespace jinja2 { +inline namespace v2 +{ class ExpressionEvaluatorBase : public ArenaNode { @@ -1064,6 +1066,7 @@ ParsedArgumentsInfo ParseCallParamsInfo(const std::vector& args, c CallParams EvaluateCallParams(const CallParamsInfo& info, RenderContext& context); CallParams EvaluateCallParams(const ArenaCallParams& info, RenderContext& context); } // namespace helpers +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_EXPRESSION_EVALUATOR_H diff --git a/src/expression_parser.cpp b/src/expression_parser.cpp index b5aba9a65..2c98d3afd 100644 --- a/src/expression_parser.cpp +++ b/src/expression_parser.cpp @@ -28,6 +28,8 @@ namespace jinja2 { +inline namespace v2 +{ ParseError::ParseError(const ParseError&) = default; ParseError::ParseError(ParseError&& other) noexcept @@ -1281,4 +1283,5 @@ ExpressionParser::ParseResult> ExpressionParser::ParseIfEx return result; } +} // namespace v2 } // namespace jinja2 diff --git a/src/expression_parser.h b/src/expression_parser.h index b4687a373..1f1504437 100644 --- a/src/expression_parser.h +++ b/src/expression_parser.h @@ -20,6 +20,8 @@ namespace jinja2 { +inline namespace v2 +{ // Jinja2 rejects a filter or test it does not know when it compiles the template, except in // an `if` statement or an inline if, which leave the check to the call (Frame.soft_frame in // jinja2/compiler.py). The expression parsers of one template report each unknown name here. @@ -211,6 +213,7 @@ class ExpressionParser unsigned m_operators = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_EXPRESSION_PARSER_H diff --git a/src/filesystem_handler.cpp b/src/filesystem_handler.cpp index c480524b4..17759ff3f 100644 --- a/src/filesystem_handler.cpp +++ b/src/filesystem_handler.cpp @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ using TargetFileStream = std::variant; @@ -277,4 +279,5 @@ bool RealFileSystem::IsEqual(const IComparable& other) const return m_rootFolder == ptr->m_rootFolder; } +} // namespace v2 } // namespace jinja2 diff --git a/src/filter_factories.cpp b/src/filter_factories.cpp index 3df04841e..dd2a5e05e 100644 --- a/src/filter_factories.cpp +++ b/src/filter_factories.cpp @@ -22,6 +22,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -164,4 +166,5 @@ FilterPtr CreateFilter(const std::string& filterName, const CallParamsInfo& para return std::make_shared(filterName, params, std::move(callable)); } +} // namespace v2 } // namespace jinja2 diff --git a/src/filters.cpp b/src/filters.cpp index 5867861eb..e6615f135 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -41,7 +41,11 @@ using namespace std::string_literals; -namespace jinja2::filters +namespace jinja2 +{ +inline namespace v2 +{ +namespace filters { // Which values Python can order against each other with `<` @@ -2604,4 +2608,6 @@ InternalValue UserDefinedFilter::Filter(const InternalValue& baseVal, RenderCont return callable->GetExpressionCallable()(callParams, context); } -} // namespace jinja2::filters +} // namespace filters +} // namespace v2 +} // namespace jinja2 diff --git a/src/filters.h b/src/filters.h index dd46467d5..b73e98964 100644 --- a/src/filters.h +++ b/src/filters.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ using FilterPtr = std::shared_ptr; using FilterParams = CallParamsInfo; @@ -340,6 +342,7 @@ class UserDefinedFilter : public FilterBase }; } // namespace filters +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_FILTERS_H diff --git a/src/function_base.h b/src/function_base.h index 2af06a502..f0bc1d502 100644 --- a/src/function_base.h +++ b/src/function_base.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ // The parameters of a filter or tester kind, built once (a function-local static): an instance // keeps a pointer to its table, so the names and defaults are stored once per kind using ArgumentsTable = std::vector; @@ -166,6 +168,7 @@ inline bool FunctionBase::ParseParams(const ArgumentsTable& argsInfo, const Call return result; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_FUNCTION_BASE_H diff --git a/src/generic_adapters.h b/src/generic_adapters.h index a7b982ffd..95313c163 100644 --- a/src/generic_adapters.h +++ b/src/generic_adapters.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ template class IndexedEnumeratorImpl : public Base @@ -331,6 +333,7 @@ inline std::optional IndexedListItemAccessorImpl::CreateEn return MakeEnumerator(this); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_GENERIC_ADAPTERS_H diff --git a/src/generic_list.cpp b/src/generic_list.cpp index cfb5b06c2..2652c907e 100644 --- a/src/generic_list.cpp +++ b/src/generic_list.cpp @@ -3,6 +3,8 @@ namespace jinja2 { +inline namespace v2 +{ GenericList::iterator GenericList::begin() const { @@ -47,4 +49,5 @@ bool operator!=(const GenericList& lhs, const GenericList& rhs) return !(lhs == rhs); } +} // namespace v2 } // namespace jinja2 diff --git a/src/global_functions.cpp b/src/global_functions.cpp index 573a1a83c..f6851bb56 100644 --- a/src/global_functions.cpp +++ b/src/global_functions.cpp @@ -28,6 +28,8 @@ using namespace std::string_literals; namespace jinja2 { +inline namespace v2 +{ namespace { // jinja2.constants.LOREM_IPSUM_WORDS @@ -530,4 +532,5 @@ const InternalValueMap& GetBuiltinGlobals(bool withI18n) }(); return withI18n ? globalsWithI18n : globals; } +} // namespace v2 } // namespace jinja2 diff --git a/src/helpers.h b/src/helpers.h index b189b84c9..ec0eab7f1 100644 --- a/src/helpers.h +++ b/src/helpers.h @@ -9,6 +9,8 @@ namespace jinja2 { +inline namespace v2 +{ struct MultiStringLiteral { const char* charValue; @@ -281,6 +283,7 @@ Sequence CompileEscapes(Sequence s, bool* ok = nullptr) return s; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_HELPERS_H diff --git a/src/html_entities.h b/src/html_entities.h index ffe3f2de9..b02240303 100644 --- a/src/html_entities.h +++ b/src/html_entities.h @@ -8,7 +8,11 @@ #include #include -namespace jinja2::html +namespace jinja2 +{ +inline namespace v2 +{ +namespace html { // A named character reference and the one or two code points it stands for (0 when only one) @@ -2261,6 +2265,8 @@ inline const HtmlEntity* FindHtmlEntity(std::string_view name) return it != std::end(HtmlEntities) && std::string_view(it->name) == name ? &*it : nullptr; } -} // namespace jinja2::html +} // namespace html +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_HTML_ENTITIES_H diff --git a/src/internal_value.cpp b/src/internal_value.cpp index 8f4405d97..03d299ffc 100644 --- a/src/internal_value.cpp +++ b/src/internal_value.cpp @@ -31,6 +31,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -2246,4 +2248,5 @@ bool Callable::IsSame(const Callable& other) const return left != nullptr && right != nullptr && left->val.IsEqual(right->val); } +} // namespace v2 } // namespace jinja2 diff --git a/src/internal_value.h b/src/internal_value.h index c27717ae7..d8e1a7162 100644 --- a/src/internal_value.h +++ b/src/internal_value.h @@ -45,6 +45,8 @@ namespace jinja2 { +inline namespace v2 +{ template class ReferenceWrapper @@ -1746,6 +1748,7 @@ static_assert(std::is_nothrow_move_assignable_v); Value IntValue2Value(const InternalValue& val); Value OptIntValue2Value(std::optional val); +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_INTERNAL_VALUE_H diff --git a/src/lexer.cpp b/src/lexer.cpp index 7a33af003..4af410a48 100644 --- a/src/lexer.cpp +++ b/src/lexer.cpp @@ -13,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -77,4 +79,5 @@ InternalValue ParseNumberLiteral(std::string number) return InternalValue(std::strtod(digits, nullptr)); } +} // namespace v2 } // namespace jinja2 diff --git a/src/lexer.h b/src/lexer.h index f932d0ce0..2596b35ae 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ struct CharRange { size_t startOffset; @@ -919,6 +921,7 @@ class LexScanner } }; +} // namespace v2 } // namespace jinja2 namespace std diff --git a/src/load_settings.h b/src/load_settings.h index 9ce584d18..684bf0dd5 100644 --- a/src/load_settings.h +++ b/src/load_settings.h @@ -11,7 +11,11 @@ #include #include -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // Delimiters and line prefixes from Settings, in the character type of a template @@ -122,6 +126,8 @@ using LoadSettingsPtr = std::shared_ptr; // For templates made without an environment const LoadSettingsPtr& DefaultLoadSettings(); -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_LOAD_SETTINGS_H diff --git a/src/lookup_result.h b/src/lookup_result.h index 8107208c5..3586d255d 100644 --- a/src/lookup_result.h +++ b/src/lookup_result.h @@ -7,6 +7,8 @@ namespace jinja2 { +inline namespace v2 +{ // Where a lookup found a value, or nothing: a nullable reference to a value stored elsewhere // (a scope, a constant node), read in place instead of copied (docs/tasks/0088, 0117). It is // valid while what holds the value is unchanged: for a variable, until a scope of the @@ -47,6 +49,7 @@ class [[nodiscard]] BasicLookupResult using LookupResult = BasicLookupResult; // A variable found for changing in place (docs/tasks/0020) using MutableLookupResult = BasicLookupResult; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_LOOKUP_RESULT_H diff --git a/src/loop_attr.h b/src/loop_attr.h index 30de75c8f..d77d45f2c 100644 --- a/src/loop_attr.h +++ b/src/loop_attr.h @@ -6,6 +6,8 @@ namespace jinja2 { +inline namespace v2 +{ // An attribute of a for loop's `loop` object. `loop.` in the template is resolved to // one of these at Load, so a render reads the attribute without comparing strings // (docs/design/perf-design-overview.md row 17; 0117 P1) @@ -61,6 +63,7 @@ inline LoopAttr FindLoopAttr(std::string_view name) } return LoopAttr::None; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_LOOP_ATTR_H diff --git a/src/make_unexpected.h b/src/make_unexpected.h index 475394766..821f760a7 100644 --- a/src/make_unexpected.h +++ b/src/make_unexpected.h @@ -7,6 +7,8 @@ namespace jinja2 { +inline namespace v2 +{ // The one place that names the backing `expected` library's error factory, so that // switching it (docs/api-2.0.md, decision 4) touches this function and not every @@ -17,6 +19,7 @@ auto MakeUnexpected(E&& error) return nonstd::make_unexpected(std::forward(error)); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_MAKE_UNEXPECTED_H diff --git a/src/markup.h b/src/markup.h index e5f34f6e9..bfb702180 100644 --- a/src/markup.h +++ b/src/markup.h @@ -16,6 +16,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -218,6 +220,7 @@ inline void WriteOutput(OutStream& stream, const InternalValue& val, RenderConte WriteEscaped(stream, val, context.GetRendererCallback()); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_MARKUP_H diff --git a/src/name_resolver.cpp b/src/name_resolver.cpp index 9c8381cf1..a783d2cf7 100644 --- a/src/name_resolver.cpp +++ b/src/name_resolver.cpp @@ -19,6 +19,8 @@ namespace jinja2 { +inline namespace v2 +{ NameResolver::FrameId NameResolver::Push(FrameId parent, Kind kind) { const auto id = static_cast(m_frames.size()); @@ -488,4 +490,5 @@ std::pair NameResolver::Find(const NodeArena& nodes, NodeRef< } return dynamic; } +} // namespace v2 } // namespace jinja2 diff --git a/src/name_resolver.h b/src/name_resolver.h index 902b1fc65..4d7ea777b 100644 --- a/src/name_resolver.h +++ b/src/name_resolver.h @@ -17,6 +17,8 @@ namespace jinja2 { +inline namespace v2 +{ class BindUndefinedStatement; class ForStatement; class IRendererBase; @@ -246,6 +248,7 @@ class NameResolver bool m_inBranch = false; bool m_hasSlots = false; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_NAME_RESOLVER_H diff --git a/src/node_arena.cpp b/src/node_arena.cpp index f5cc922da..d039dd8dd 100644 --- a/src/node_arena.cpp +++ b/src/node_arena.cpp @@ -20,6 +20,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { void OffsetList::Grow() @@ -259,4 +261,5 @@ SealedArena NodeArena::SealWith(boost::span roots) } return sealed; } +} // namespace v2 } // namespace jinja2 diff --git a/src/node_arena.h b/src/node_arena.h index 2caba5c85..fc0aa055a 100644 --- a/src/node_arena.h +++ b/src/node_arena.h @@ -37,6 +37,8 @@ namespace jinja2 { +inline namespace v2 +{ // How much a handle is checked before it is followed (CMake JINJA2CPP_NODEREF_CHECKS): // 0 (OFF): never. 1 (ON, the Release default): Seal checks every handle of the tree, and // a render checks the ones that cross into another template. 2 (FULL, Debug, sanitizer and @@ -1392,6 +1394,7 @@ class NodeArena : public detail::ArenaAccess std::size_t m_symbolCount = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_NODE_ARENA_H diff --git a/src/out_stream.cpp b/src/out_stream.cpp index d64adf61d..039d07b5f 100644 --- a/src/out_stream.cpp +++ b/src/out_stream.cpp @@ -13,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ void OutStream::Flush() { if (!m_writer) @@ -262,4 +264,5 @@ void OutStream::WriteValueTo(const InternalValue& val) template void OutStream::WriteValueTo(const InternalValue& val); template void OutStream::WriteValueTo(const InternalValue& val); +} // namespace v2 } // namespace jinja2 diff --git a/src/out_stream.h b/src/out_stream.h index a080c4de8..432e41b2e 100644 --- a/src/out_stream.h +++ b/src/out_stream.h @@ -11,6 +11,8 @@ namespace jinja2 { +inline namespace v2 +{ // Where a render writes its output. A render to a string, the common case, gathers the // output in a small buffer and appends it to the string a buffer at a time: a fragment // costs a bounds check and a memcpy, with no virtual call (docs/tasks/0113). Any other @@ -171,6 +173,7 @@ TargetString RenderToString(Callback* callback, Fn&& render) return result; } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_OUT_STREAM_H diff --git a/src/python_format.cpp b/src/python_format.cpp index c6739a14c..ca83ad2bd 100644 --- a/src/python_format.cpp +++ b/src/python_format.cpp @@ -21,6 +21,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -864,4 +866,5 @@ std::string CompiledPercentFormat::Format(const InternalValue* args, size_t coun return Formatter(args, count, false).Run(m_format, m_pieces); } +} // namespace v2 } // namespace jinja2 diff --git a/src/python_format.h b/src/python_format.h index 6c1e35dc8..77487aefa 100644 --- a/src/python_format.h +++ b/src/python_format.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ // Python's printf-style formatting, `format % values`, for a UTF-8 format string. `values` is // what Python's right operand is: a tuple (a list marked as tuple) of positional arguments, a @@ -78,6 +80,7 @@ class CompiledPercentFormat std::vector m_pieces; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_PYTHON_FORMAT_H diff --git a/src/recursion_guard.cpp b/src/recursion_guard.cpp index df22bb284..6716709a4 100644 --- a/src/recursion_guard.cpp +++ b/src/recursion_guard.cpp @@ -29,6 +29,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { // Stack left free for the work between two checks: a filter, a value conversion, the @@ -104,4 +106,5 @@ bool StackNearlyExhaustedSlow(std::uintptr_t frame) } } // namespace detail +} // namespace v2 } // namespace jinja2 diff --git a/src/recursion_guard.h b/src/recursion_guard.h index 4848d66cf..6f8c617b9 100644 --- a/src/recursion_guard.h +++ b/src/recursion_guard.h @@ -11,6 +11,8 @@ namespace jinja2 { +inline namespace v2 +{ // Limits that keep hostile or runaway templates from overflowing the stack (docs/tasks/0003). // @@ -189,6 +191,7 @@ class RenderDepthGuard DepthGuard m_guard; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RECURSION_GUARD_H diff --git a/src/render_context.h b/src/render_context.h index f4b56a657..9bb7216f9 100644 --- a/src/render_context.h +++ b/src/render_context.h @@ -51,6 +51,8 @@ namespace jinja2 { +inline namespace v2 +{ template class TemplateImpl; @@ -1987,6 +1989,7 @@ class HomeCall int m_debugExceptions = 0; #endif }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RENDER_CONTEXT_H diff --git a/src/render_workspace.cpp b/src/render_workspace.cpp index dc4c98897..714f1eb86 100644 --- a/src/render_workspace.cpp +++ b/src/render_workspace.cpp @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ RenderWorkspace::RenderWorkspace() { m_loopFrames.reserve(MaxLoopFrames); @@ -70,4 +72,5 @@ void RenderWorkspace::EmptyMaps(RenderMaps& maps, bool keepTables) noexcept } } } +} // namespace v2 } // namespace jinja2 diff --git a/src/render_workspace.h b/src/render_workspace.h index d6cace44b..8b9408d84 100644 --- a/src/render_workspace.h +++ b/src/render_workspace.h @@ -16,6 +16,8 @@ namespace jinja2 { +inline namespace v2 +{ struct GlobalsSnapshot; namespace detail @@ -171,6 +173,7 @@ class RenderWorkspace std::vector> m_maps; std::size_t m_lentMaps = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RENDER_WORKSPACE_H diff --git a/src/renderer.h b/src/renderer.h index 4368339d9..7ba7cfef3 100644 --- a/src/renderer.h +++ b/src/renderer.h @@ -18,6 +18,8 @@ namespace jinja2 { +inline namespace v2 +{ class IRendererBase : public ArenaNode { public: @@ -181,6 +183,7 @@ inline NodeRef MakeExpressionRenderer(NodeArena& nodes, NodeRef(expr); } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_RENDERER_H diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index 71bc148a0..315ff9289 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -30,7 +30,11 @@ using namespace std::string_literals; -namespace jinja2::filters +namespace jinja2 +{ +inline namespace v2 +{ +namespace filters { struct PrettyPrinter : visitors::BaseVisitor { @@ -620,4 +624,6 @@ InternalValue XmlAttrFilter::Filter(const InternalValue& baseVal, RenderContext& return resultVal; } -} // namespace jinja2::filters +} // namespace filters +} // namespace v2 +} // namespace jinja2 diff --git a/src/slot_frame.h b/src/slot_frame.h index 037bea479..b134123c4 100644 --- a/src/slot_frame.h +++ b/src/slot_frame.h @@ -25,6 +25,8 @@ namespace jinja2 { +inline namespace v2 +{ // A name's slot in the frame of its unit; Dynamic for a name looked up by name struct SlotIndex { @@ -152,6 +154,7 @@ struct ScopeView bool operator==(const ScopeView& other) const { return view == other.view && scopeIndex == other.scopeIndex && linkBelow == other.linkBelow; } bool operator!=(const ScopeView& other) const { return !(*this == other); } }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_SLOT_FRAME_H diff --git a/src/statements.cpp b/src/statements.cpp index 6c40b51c9..6f6e4a6a7 100644 --- a/src/statements.cpp +++ b/src/statements.cpp @@ -47,6 +47,8 @@ using namespace std::string_literals; namespace jinja2 { +inline namespace v2 +{ void ForStatement::Render(OutStream& os, RenderContext& values) { @@ -2642,4 +2644,5 @@ void BindUndefinedStatement::Render(OutStream& /*os*/, RenderContext& values) } } } +} // namespace v2 } // namespace jinja2 diff --git a/src/statements.h b/src/statements.h index 82c7532db..32da9d31c 100644 --- a/src/statements.h +++ b/src/statements.h @@ -22,6 +22,8 @@ namespace jinja2 { +inline namespace v2 +{ class Statement : public IRendererBase { public: @@ -962,6 +964,7 @@ class BindUndefinedStatement final : public Statement ArenaSpan m_names; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_STATEMENTS_H diff --git a/src/string_builder.h b/src/string_builder.h index 2278b1437..752e2876c 100644 --- a/src/string_builder.h +++ b/src/string_builder.h @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ // The text of a string value being built: the first 500 characters on the stack, so that the // finished value allocates once, its object holding the characters (docs/tasks/0151). Code that @@ -64,6 +66,7 @@ class StringBuilder fmt::basic_memory_buffer m_buffer; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_STRING_BUILDER_H diff --git a/src/string_converter_filter.cpp b/src/string_converter_filter.cpp index 58f23017c..27b205cff 100644 --- a/src/string_converter_filter.cpp +++ b/src/string_converter_filter.cpp @@ -35,7 +35,11 @@ namespace ba = boost::algorithm; using namespace std::string_literals; -namespace jinja2::filters +namespace jinja2 +{ +inline namespace v2 +{ +namespace filters { template @@ -2035,9 +2039,15 @@ InternalValue StringConverter::Filter(const InternalValue& baseVal, RenderContex return resultVal; } -} // namespace jinja2::filters +} // namespace filters +} // namespace v2 +} // namespace jinja2 -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { namespace { @@ -2091,4 +2101,6 @@ int CompareLowerUnicode(std::wstring_view left, std::wstring_view right) { return CompareLower(left, right); } -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 diff --git a/src/template.cpp b/src/template.cpp index 07103b2c8..75e7d7312 100644 --- a/src/template.cpp +++ b/src/template.cpp @@ -20,6 +20,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { template @@ -182,4 +184,5 @@ template class BasicTemplate; template class BasicTemplate; #endif +} // namespace v2 } // namespace jinja2 diff --git a/src/template_env.cpp b/src/template_env.cpp index 312c9f7d5..a02a5ba1f 100644 --- a/src/template_env.cpp +++ b/src/template_env.cpp @@ -26,6 +26,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -487,4 +489,5 @@ void TemplateEnv::ApplyGlobals(const std::function& fn) fn(*m_impl->globalValues); } +} // namespace v2 } // namespace jinja2 diff --git a/src/template_env_impl.h b/src/template_env_impl.h index e39adf31c..176759c19 100644 --- a/src/template_env_impl.h +++ b/src/template_env_impl.h @@ -19,7 +19,11 @@ #include #include -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // The state of a TemplateEnv. The TemplateEnv the user creates owns it; every template made in the environment holds @@ -100,6 +104,8 @@ struct TemplateEnvAccess static TemplateEnv MakeHandle(const TemplateEnv* env) { return TemplateEnv(env ? env->m_impl : nullptr); } }; -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_ENV_IMPL_H diff --git a/src/template_impl.h b/src/template_impl.h index d3538f202..9fa80f3b5 100644 --- a/src/template_impl.h +++ b/src/template_impl.h @@ -63,6 +63,8 @@ namespace jinja2 { +inline namespace v2 +{ // The default globals of every render, looked up after the environment's (global_functions.cpp) extern const InternalValueMap& GetBuiltinGlobals(bool withI18n); @@ -827,6 +829,7 @@ class TemplateImpl : public ITemplateImpl MetadataInfo m_metadataInfo; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_IMPL_H diff --git a/src/template_parser.cpp b/src/template_parser.cpp index 88c0723bc..eb4a4879f 100644 --- a/src/template_parser.cpp +++ b/src/template_parser.cpp @@ -34,6 +34,8 @@ namespace jinja2 { +inline namespace v2 +{ NodeRef StatementsParser::TakeFrameBody(StatementInfo& info) { @@ -1943,4 +1945,5 @@ InternalValue EscapedLiteralValue(std::basic_string_view text, const std: template InternalValue EscapedLiteralValue(std::string_view text, const std::string& newlineSequence, bool& malformed); template InternalValue EscapedLiteralValue(std::wstring_view text, const std::string& newlineSequence, bool& malformed); +} // namespace v2 } // namespace jinja2 diff --git a/src/template_parser.h b/src/template_parser.h index 787960f41..fe0cc7625 100644 --- a/src/template_parser.h +++ b/src/template_parser.h @@ -50,6 +50,8 @@ namespace jinja2 { +inline namespace v2 +{ template struct ParserTraits; @@ -2391,6 +2393,7 @@ std::unordered_map ParserTraitsBase::s_tokens = { { Token::ExprEnd, UNIVERSAL_STR("}}") }, }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_PARSER_H diff --git a/src/template_slots.h b/src/template_slots.h index 21f1781f9..3d755f6b4 100644 --- a/src/template_slots.h +++ b/src/template_slots.h @@ -15,6 +15,8 @@ namespace jinja2 { +inline namespace v2 +{ // A template one render runs, by its place in the render's table of templates (0118 P4b). // The render keeps every template it loads until it ends, so a handle needs no reference @@ -165,6 +167,7 @@ class TemplateSlots std::uint8_t m_generation = 0; }; +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TEMPLATE_SLOTS_H diff --git a/src/tester_factories.cpp b/src/tester_factories.cpp index 2a3c452ba..8136a2b80 100644 --- a/src/tester_factories.cpp +++ b/src/tester_factories.cpp @@ -22,6 +22,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -149,4 +151,5 @@ TesterPtr CreateTester(std::string testerName, const CallParamsInfo& params, Ren return std::make_shared(std::move(testerName), params, std::move(callable)); } +} // namespace v2 } // namespace jinja2 diff --git a/src/testers.cpp b/src/testers.cpp index caba25367..a44b4af05 100644 --- a/src/testers.cpp +++ b/src/testers.cpp @@ -23,7 +23,11 @@ using namespace std::string_literals; -namespace jinja2::testers +namespace jinja2 +{ +inline namespace v2 +{ +namespace testers { Comparator::Comparator(const TesterParams& params, BinaryExpression::Operation op) @@ -598,4 +602,6 @@ bool UserDefinedTester::Test(const InternalValue& baseVal, RenderContext& contex return ConvertToBool(callable->GetExpressionCallable()(callParams, context)); } -} // namespace jinja2::testers +} // namespace testers +} // namespace v2 +} // namespace jinja2 diff --git a/src/testers.h b/src/testers.h index 37f906679..0c2e8298c 100644 --- a/src/testers.h +++ b/src/testers.h @@ -13,6 +13,8 @@ namespace jinja2 { +inline namespace v2 +{ using TesterPtr = std::shared_ptr; using TesterParams = CallParamsInfo; @@ -117,6 +119,7 @@ class UserDefinedTester : public TesterBase InternalValue m_callable; }; } // namespace testers +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_TESTERS_H diff --git a/src/undefined.cpp b/src/undefined.cpp index a397e1107..ce865586c 100644 --- a/src/undefined.cpp +++ b/src/undefined.cpp @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { @@ -132,4 +134,5 @@ const char* UndefinedClassName(const UndefinedInfo& info) return "Undefined"; } +} // namespace v2 } // namespace jinja2 diff --git a/src/undefined.h b/src/undefined.h index a1ae94598..a7b1c614e 100644 --- a/src/undefined.h +++ b/src/undefined.h @@ -12,6 +12,8 @@ namespace jinja2 { +inline namespace v2 +{ // What a named lookup missed, and the policy that decides how the undefined result may be // used. It travels with the value, as the class of a Python Undefined does: a variable set @@ -102,6 +104,7 @@ inline void CheckStrictUndefined(const UndefinedValue& val) } } +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_UNDEFINED_H diff --git a/src/unicode_case.h b/src/unicode_case.h index 75a163e37..0e6d46c9d 100644 --- a/src/unicode_case.h +++ b/src/unicode_case.h @@ -10,7 +10,11 @@ #include #include -namespace jinja2::unicode +namespace jinja2 +{ +inline namespace v2 +{ +namespace unicode { // cp + delta for every stride-th code point from first to last @@ -1413,6 +1417,8 @@ inline size_t ToTitle(uint32_t cp, uint32_t (&out)[3]) { return MapCase(TitleRun inline bool IsCased(uint32_t cp) { return InRanges(CasedRanges, cp); } inline bool IsCaseIgnorable(uint32_t cp) { return InRanges(CaseIgnorableRanges, cp); } -} // namespace jinja2::unicode +} // namespace unicode +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_UNICODE_CASE_H diff --git a/src/unicode_printable.h b/src/unicode_printable.h index 8162d7607..8fa21ec43 100644 --- a/src/unicode_printable.h +++ b/src/unicode_printable.h @@ -7,7 +7,11 @@ #include #include -namespace jinja2::detail +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail { // True when Python's str.isprintable() holds for the code point, so repr() keeps it as is @@ -216,6 +220,8 @@ inline bool IsPythonPrintable(uint32_t cp) return cp > it->last; } -} // namespace jinja2::detail +} // namespace detail +} // namespace v2 +} // namespace jinja2 #endif // JINJA2CPP_SRC_UNICODE_PRINTABLE_H diff --git a/src/unicode_tables.h b/src/unicode_tables.h index b43614642..fce4ba143 100644 --- a/src/unicode_tables.h +++ b/src/unicode_tables.h @@ -9,6 +9,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace unicode { @@ -877,6 +879,7 @@ inline bool IsSpace(uint32_t cp) } } // namespace unicode +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_UNICODE_TABLES_H diff --git a/src/value.cpp b/src/value.cpp index 9594ad940..b6824083c 100644 --- a/src/value.cpp +++ b/src/value.cpp @@ -14,6 +14,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace { // Every printing form of Value meets here, so none can print differently from {{ value }}: @@ -115,4 +117,5 @@ std::to_chars_result Value::ToChars(char* first, char* last) const noexcept } } +} // namespace v2 } // namespace jinja2 diff --git a/src/value_methods.cpp b/src/value_methods.cpp index f098d3879..39d77bc39 100644 --- a/src/value_methods.cpp +++ b/src/value_methods.cpp @@ -31,7 +31,11 @@ using namespace std::string_literals; -namespace jinja2::methods +namespace jinja2 +{ +inline namespace v2 +{ +namespace methods { namespace { @@ -2388,4 +2392,6 @@ void ThrowNoAttribute(const InternalValue& obj, const std::string& name) Raise("'" + TypeName(obj) + " object' has no attribute '" + name + "'"); } -} // namespace jinja2::methods +} // namespace methods +} // namespace v2 +} // namespace jinja2 diff --git a/src/value_methods.h b/src/value_methods.h index 9d6f2df47..ff9283195 100644 --- a/src/value_methods.h +++ b/src/value_methods.h @@ -8,6 +8,8 @@ namespace jinja2 { +inline namespace v2 +{ struct CallParams; // Python's builtin methods on template values: s.upper(), l.append(x), d.items(), ... @@ -65,6 +67,7 @@ bool IsContainer(const InternalValue& value); [[noreturn]] void ThrowNoAttribute(const InternalValue& obj, const std::string& name); } // namespace methods +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_VALUE_METHODS_H diff --git a/src/value_visitors.h b/src/value_visitors.h index 982d139ff..a3ac1af01 100644 --- a/src/value_visitors.h +++ b/src/value_visitors.h @@ -37,6 +37,8 @@ namespace jinja2 { +inline namespace v2 +{ namespace detail { @@ -2069,6 +2071,7 @@ JINJA2CPP_HOT_INLINE InternalValue ApplyToNumbers(const InternalValue& left, con } // namespace visitors +} // namespace v2 } // namespace jinja2 #endif // JINJA2CPP_SRC_VALUE_VISITORS_H From a55332875f404f893195e387ce38e1e1bc937771 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 00:53:06 +0000 Subject: [PATCH 12/18] ABI gates for the inline namespace v2: nm test, link probes, SONAME, abidiff (0076) - format-check runs scripts/abi_namespace.py --check - CTest jinja2cpp_abi_namespace: no jinja2 symbol outside v2, no global xyz - test/package: abi_probe_v2 links, abi_probe_v1 (1.x mangling) must not - check_package.cmake checks the installed SONAME; version asserts need major 2 - linux-build: one shared RelWithDebInfo row per standard dumps the ABI with abidw and reports abidiff against the last master build (baselines at the RC, task 0285) - README: Compatibility and versioning; find_package(jinja2cpp 2) Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Ua8ABM1TRsAvso2sWGHFSw --- .git-blame-ignore-revs | 3 + .github/workflows/format-check.yml | 5 ++ .github/workflows/linux-build.yml | 51 +++++++++++- CMakeLists.txt | 6 ++ README.md | 23 +++++- abi/suppressions.ini | 22 +++++ docs/tasks/0076-abi-namespace-and-headers.md | 16 +++- docs/tasks/0285-abi-baselines-at-rc.md | 26 ++++++ .../polymorphic_value/indirect_cxx14.h | 2 +- .../polymorphic_value/polymorphic_cxx14.h | 2 +- sbom/components.json | 18 ++++ sbom/jinja2cpp-full.cdx.json | 73 +++++++++++++++-- sbom/jinja2cpp-full.spdx.json | 36 ++++++-- sbom/jinja2cpp.cdx.json | 16 ++-- sbom/jinja2cpp.spdx.json | 10 +-- scripts/abi_report.sh | 82 +++++++++++++++++++ test/abi_symbols.cmake | 48 +++++++++++ test/package/CMakeLists.txt | 15 ++++ test/package/abi_probe.cpp | 28 +++++++ test/package/check_package.cmake | 22 +++++ test/package/main.cpp | 5 +- test/template_api_test.cpp | 3 +- 22 files changed, 475 insertions(+), 37 deletions(-) create mode 100644 abi/suppressions.ini create mode 100644 docs/tasks/0285-abi-baselines-at-rc.md create mode 100755 scripts/abi_report.sh create mode 100644 test/abi_symbols.cmake create mode 100644 test/package/abi_probe.cpp diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c1dbb6264..aadb81db3 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -4,3 +4,6 @@ # docs/tasks/0060: braces around every single-statement body (clang-format InsertBraces) 33b30998156056c4de59dcb0ce598390602ab253 + +# docs/tasks/0076: namespace jinja2 wrapped in inline namespace v2 (scripts/abi_namespace.py) +ae4649c98baaf1999e99096f7d39ef7a4dca2b8d diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index c7f97435f..876aeb361 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -30,6 +30,11 @@ jobs: - name: Check formatting of src/ and include/ run: git ls-files 'src/**.h' 'src/**.cpp' 'include/**.h' | xargs clang-format-18 --dry-run --Werror + # Every namespace jinja2 in src/ and include/ opens inline namespace v2, the ABI tag + # (docs/tasks/0076); the nm test checks the symbols, this also covers header-only code + - name: Check the inline ABI namespace + run: python3 scripts/abi_namespace.py --check + # test/ still has hand-aligned tables, so only lines touched by the pull request # are checked there. src/ and include/ are left to the whole-file step above: given # a vendored file, git clang-format reports it as emptied (.clang-format-ignore). diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index e2cdfc8f0..a75d72278 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -46,6 +46,11 @@ jobs: # Two rows also install the library and build a consumer against the installed package # at another standard (`package`, docs/tasks/0068): a static and a shared install, each # read through find_package and pkg-config. It is a check after the tests, not an axis. + # One shared row per standard builds RelWithDebInfo (`abi`) so that abidiff can compare + # the library's exported ABI with the previous one (docs/tasks/0076): the headers differ + # by standard, so the ABI is checked once per standard. For the pairwise design those rows + # count as Release (optimised); the remaining Release rows still meet every standard, + # compiler, linkage, binding and noderef value on their own. linux-build: name: C++${{ matrix.std }} ${{ matrix.compiler }} ${{ matrix.config }} shared=${{ matrix.shared }} json=${{ matrix.bindings }}${{ matrix.noderef == 'OFF' && ' noderef=OFF' || '' }} runs-on: ubuntu-24.04 @@ -56,14 +61,14 @@ jobs: include: - { std: 17, compiler: gcc-12, config: Release, shared: 'OFF', bindings: boost, noderef: 'OFF' } - { std: 17, compiler: gcc-14, config: Debug, shared: 'OFF', bindings: rapid } - - { std: 17, compiler: clang-18, config: Release, shared: 'ON', bindings: nlohmann } + - { std: 17, compiler: clang-18, config: RelWithDebInfo, shared: 'ON', bindings: nlohmann, abi: true } - { std: 17, compiler: clang-20, config: Release, shared: 'OFF', bindings: boost, package: 23 } - - { std: 20, compiler: gcc-12, config: Release, shared: 'ON', bindings: rapid } + - { std: 20, compiler: gcc-12, config: RelWithDebInfo, shared: 'ON', bindings: rapid, abi: true } - { std: 20, compiler: gcc-14, config: Release, shared: 'ON', bindings: nlohmann } - { std: 20, compiler: clang-18, config: Debug, shared: 'OFF', bindings: boost } - { std: 20, compiler: clang-20, config: Debug, shared: 'OFF', bindings: nlohmann, noderef: 'OFF' } - { std: 23, compiler: gcc-12, config: Debug, shared: 'OFF', bindings: nlohmann } - - { std: 23, compiler: gcc-14, config: Release, shared: 'ON', bindings: boost, noderef: 'OFF', package: 17 } + - { std: 23, compiler: gcc-14, config: RelWithDebInfo, shared: 'ON', bindings: boost, noderef: 'OFF', package: 17, abi: true } - { std: 23, compiler: clang-18, config: Release, shared: 'ON', bindings: rapid, noderef: 'OFF' } - { std: 23, compiler: clang-20, config: Debug, shared: 'ON', bindings: rapid } - { std: 26, compiler: gcc-14, config: Release, shared: 'OFF', bindings: boost } @@ -99,6 +104,46 @@ jobs: - name: Test run: ctest --test-dir .build --output-on-failure + # abidiff against the ABI of the previous master build (report only) or, once the RC + # commits abi/libjinja2cpp.so.2.cxx.abi, against that baseline: an incompatible + # change then fails unless SOVERSION is bumped and the baseline regenerated (0076) + - name: ABI dump + if: matrix.abi + run: | + sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends abigail-tools + mkdir -p .abi + abidw --headers-dir include --drop-private-types --no-show-locs \ + --suppressions abi/suppressions.ini \ + --out-file .abi/libjinja2cpp.so.2.cxx${{ matrix.std }}.abi .build/libjinja2cpp.so + + - name: Previous ABI dump + if: matrix.abi && github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: .abi-base + key: abi-cxx${{ matrix.std }}-${{ github.event.pull_request.base.sha }} + restore-keys: abi-cxx${{ matrix.std }}- + + - name: ABI report + if: matrix.abi + run: scripts/abi_report.sh ${{ matrix.std }} + + - name: Save ABI dump + if: matrix.abi && github.event_name == 'push' + run: mkdir -p .abi-base && cp .abi/*.abi .abi-base/ + + - if: matrix.abi && github.event_name == 'push' + uses: actions/cache/save@v4 + with: + path: .abi-base + key: abi-cxx${{ matrix.std }}-${{ github.sha }} + + - if: matrix.abi + uses: actions/upload-artifact@v4 + with: + name: abi-cxx${{ matrix.std }} + path: .abi/ + - name: Installed package if: matrix.package run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index fc21e80f1..5321c38ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -454,6 +454,12 @@ if (JINJA2CPP_BUILD_TESTS) add_test(NAME jinja2cpp_tests COMMAND jinja2cpp_tests --gtest_filter=-Parity*) # Differential corpus against Python Jinja2, see test/parity/README.md add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*) + # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076) + if (CMAKE_NM AND NOT MSVC) + add_test(NAME jinja2cpp_abi_namespace + COMMAND ${CMAKE_COMMAND} -DNM=${CMAKE_NM} -DLIBRARY=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/test/abi_symbols.cmake) + endif () if ("${JINJA2CPP_WITH_JSON_BINDINGS}" STREQUAL "boost") set(_bindings_define "-DJINJA2CPP_WITH_JSON_BINDINGS_BOOST") diff --git a/README.md b/README.md index 3b63ff5c1..716818956 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ The install carries a CMake package and a pkg-config file. With CMake, point target: ```cmake -find_package(jinja2cpp 1 REQUIRED) +find_package(jinja2cpp 2 REQUIRED) target_link_libraries(my_app PRIVATE jinja2cpp::jinja2cpp) ``` @@ -201,6 +201,27 @@ it in the default internal dependencies mode). The un-namespaced `jinja2cpp` tar `lib/cmake/jinja2cpp` (was `lib/jinja2cpp`), the static library in `lib` (was `lib/static`) and `jinja2cpp.pc` in `lib/pkgconfig` (was `share/pkgconfig`). Without CMake, use `pkg-config --cflags --libs jinja2cpp` (`--static` for a static build). + +### Compatibility and versioning + +- **Names.** The 1.x names that 2.0 could keep still compile, marked `[[deprecated]]`; + they are removed in 3.0. +- **ABI across majors.** Everything is declared in `jinja2::v2`, an inline namespace: you + keep writing `jinja2::Value`, but the major version is part of every mangled name, so a + program compiled against 1.x headers fails to link with a 2.x library (static or shared) + instead of misbehaving at run time. Never spell `v2` yourself. To declare Jinja2C++ types + without their definitions, include ``: a hand-written + `namespace jinja2 { class Value; }` declares a different, undefined class and makes later + uses ambiguous. `` includes the whole API except the JSON bindings. + If your code says `using namespace jinja2;` and has its own namespace-scope `v2`, qualify + one of them. +- **ABI within 2.x.** The shared library's SONAME carries the ABI version + (`libjinja2cpp.so.2`, `libjinja2cpp.2.dylib`). It is bumped by any release that changes the + layout of a type you allocate (a `Settings` field, a `Value` alternative) or the signature + of an exported function, so a shared consumer is never loaded against an incompatible + library. A static consumer has no such guard and rebuilds after a minor upgrade; patch + releases keep the ABI. + ### Usage with conan.io dependency manager Jinja2C++ can be used as conan.io package. In this case, you should do the following steps: diff --git a/abi/suppressions.ini b/abi/suppressions.ini new file mode 100644 index 000000000..27f5f5e65 --- /dev/null +++ b/abi/suppressions.ini @@ -0,0 +1,22 @@ +# libabigail suppressions for the abidiff job (.github/workflows/linux-build.yml, docs/tasks/0076). +# Only vendored internals that no public signature exposes are suppressed. jinja2::v2::detail +# is not: inline functions in the public headers call exported detail functions, so those +# are ABI too. + +# jbcoe/value_types, vendored as jinja2::v2::detail::xyz; header-only templates, their +# layout reaches users only through Value, which is reported on its own +[suppress_function] + name_regexp = ^jinja2::v2::detail::xyz:: + +[suppress_variable] + name_regexp = ^jinja2::v2::detail::xyz:: + +# robin_hood.h and lexertk.h are private to src/ and hidden by -fvisibility=hidden +[suppress_type] + name_regexp = ^(robin_hood|lexertk):: + +[suppress_function] + name_regexp = ^(robin_hood|lexertk):: + +[suppress_variable] + name_regexp = ^(robin_hood|lexertk):: diff --git a/docs/tasks/0076-abi-namespace-and-headers.md b/docs/tasks/0076-abi-namespace-and-headers.md index cbd7b3bf9..f0ae981e9 100644 --- a/docs/tasks/0076-abi-namespace-and-headers.md +++ b/docs/tasks/0076-abi-namespace-and-headers.md @@ -1,9 +1,9 @@ --- -status: in-progress +status: done priority: high area: release depends: [0072, 0073, 0074, 0075, 0043] -touches: [include/, src/, CMakeLists.txt, cmake/public/, test/package/, test/fwd_test.cpp, test/value_ptr_test.cpp, README.md, sbom/, scripts/abi_namespace.py, .git-blame-ignore-revs, .github/workflows/format-check.yml] +touches: [include/, src/, CMakeLists.txt, cmake/public/, test/package/, test/fwd_test.cpp, test/value_ptr_test.cpp, test/abi_symbols.cmake, test/template_api_test.cpp, README.md, sbom/, abi/, scripts/abi_namespace.py, scripts/abi_report.sh, .git-blame-ignore-revs, .github/workflows/format-check.yml, .github/workflows/linux-build.yml] --- # 2.0: inline ABI namespace, header layout, version @@ -36,3 +36,15 @@ future 2.x build is binary-compatible. Settle the API-shaping decisions listed i script-generated `inline namespace v2` commit, the nm/link-probe/SONAME gates, the abidiff job and the README policy. Follow-ups: 0259 (robin_hood/lexertk namespaces), 0260 (``), 0261 (Windows package link check). + +**Done.** PR-1 #580. PR-2: `scripts/abi_namespace.py` wrote `inline namespace v2` into +every `namespace jinja2` of include/ and src/ (one commit, in .git-blame-ignore-revs) and +checks it in the format-check job; CTest `jinja2cpp_abi_namespace` (test/abi_symbols.cmake) +fails on a jinja2 symbol outside `v2` or a global `xyz`; test/package links +`abi_probe_v2` and expects `abi_probe_v1` (1.x mangling) not to link, in the static and +shared package rows; check_package.cmake checks the SONAME; the version asserts require +major 2; README "Compatibility and versioning". The abidiff job runs on one shared +RelWithDebInfo row per standard and reports against the last master build; committing the +baselines at the RC is 0285. Deliberate divergence from the plan: `jinja2::v2::detail` is +not suppressed in abidiff (exported detail functions are called from inline headers), and +the C++20 abi row is gcc-12, not gcc-14, so that the Release rows keep every pair. diff --git a/docs/tasks/0285-abi-baselines-at-rc.md b/docs/tasks/0285-abi-baselines-at-rc.md new file mode 100644 index 000000000..82cb6ada8 --- /dev/null +++ b/docs/tasks/0285-abi-baselines-at-rc.md @@ -0,0 +1,26 @@ +--- +status: open +priority: high +area: release +depends: [0076, 0221] +touches: [abi/, abi/suppressions.ini] +--- +# Commit the ABI baselines at the 2.0 RC and turn the abidiff gate on + +**Problem.** Since 0076 the three `abi` rows of linux-build.yml (one shared RelWithDebInfo +row per standard) dump the library's ABI with abidw and compare it with the last master +build. Until baselines are committed the job only reports: the public API is still +changing in the API wave, and failing on every change would block it. The SOVERSION +policy (README, "Compatibility and versioning") is enforced only once +`abi/libjinja2cpp.so.2.cxx{17,20,23}.abi` exist; then `scripts/abi_report.sh` fails an +incompatible change unless SOVERSION was bumped. + +**Proposal.** On the RC commit (0221 G1 freeze), download the `abi-cxx17`, `abi-cxx20` and +`abi-cxx23` artifacts of that commit's linux-build run and commit them as `abi/`. Before +that, read one report and review `abi/suppressions.ini`: it suppresses only the vendored +value_types and robin_hood/lexertk names; `jinja2::v2::detail` stays checked because +inline header code calls exported detail functions. Add suppressions only for noise that +is shown not to reach users. + +**Done when.** The baselines are on master, a deliberate test change to an exported +layout makes the abi rows red, and docs/release-2.0.md lists the step in its RC procedure. diff --git a/include/jinja2cpp/polymorphic_value/indirect_cxx14.h b/include/jinja2cpp/polymorphic_value/indirect_cxx14.h index ad16920f1..89b27cdf4 100644 --- a/include/jinja2cpp/polymorphic_value/indirect_cxx14.h +++ b/include/jinja2cpp/polymorphic_value/indirect_cxx14.h @@ -30,7 +30,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // Local change: upstream includes "feature_check.h" here for // XYZ_HAS_TEMPLATE_ARGUMENT_DEDUCTION; the C++17 floor always has it, and this // copy stays a single header. Vendored from jbcoe/value_types 57f2e7a (task 0072). -// Local change (task 0076): namespace xyz moved into jinja2::detail and the +// Local change (task 0076): namespace xyz moved into jinja2::v2::detail and the // XYZ_ macros renamed JINJA2CPP_XYZ_, so a user's own copy of value_types // neither shadows this one through the shared include guards nor collides // with it at link time. diff --git a/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h b/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h index 6ea3042d2..489ae76f9 100644 --- a/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h +++ b/include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h @@ -29,7 +29,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // Vendored from jbcoe/value_types 57f2e7a (compatibility/polymorphic_cxx14.h) // with these local changes: -// - namespace xyz moved into jinja2::detail and the XYZ_ macros renamed +// - namespace xyz moved into jinja2::v2::detail and the XYZ_ macros renamed // JINJA2CPP_XYZ_ (task 0076), so a user's own copy of value_types neither // shadows this one through the shared include guards nor collides with it at // link time; diff --git a/sbom/components.json b/sbom/components.json index 1a33e1033..c2012833a 100644 --- a/sbom/components.json +++ b/sbom/components.json @@ -571,6 +571,24 @@ "lighten": "Not part of any build", "evidence": [{"file": ".github/workflows/format-check.yml", "contains": ["clang-format-18"]}] }, + { + "id": "libabigail", + "name": "libabigail (abigail-tools)", + "version": "2.4", + "license": "Apache-2.0 WITH LLVM-exception", + "supplier": "Red Hat / sourceware.org", + "url": "https://sourceware.org/libabigail/", + "purl": "pkg:generic/libabigail@2.4", + "delivery": "system", + "role": "tooling", + "phases": ["ci"], + "exposure": "none", + "selected_by": "linux-build CI abi rows (apt abigail-tools, Ubuntu 24.04)", + "used_for": "ABI dump and comparison of the shared library (abidw, abidiff)", + "why": "SOVERSION policy gate (docs/tasks/0076)", + "lighten": "Not part of any build", + "evidence": [{"file": ".github/workflows/linux-build.yml", "contains": ["abigail-tools"]}] + }, { "id": "cmake", "name": "CMake", diff --git a/sbom/jinja2cpp-full.cdx.json b/sbom/jinja2cpp-full.cdx.json index f241ed167..0af193245 100644 --- a/sbom/jinja2cpp-full.cdx.json +++ b/sbom/jinja2cpp-full.cdx.json @@ -1,7 +1,7 @@ { "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:3496cdbc-cf00-54e8-9a51-38efe8127a5d", + "serialNumber": "urn:uuid:05ea2675-8382-5679-ad05-ef9a8b7bf331", "version": 1, "metadata": { "timestamp": "2026-10-18T00:00:00Z", @@ -1163,11 +1163,11 @@ }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3" + "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a" }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" } ] }, @@ -1229,15 +1229,15 @@ }, { "name": "jinja2cpp:file", - "value": "src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc" + "value": "src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d" }, { "name": "jinja2cpp:file", - "value": "src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b" + "value": "src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0" }, { "name": "jinja2cpp:file", - "value": "src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3" + "value": "src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243" }, { "name": "jinja2cpp:generator", @@ -1265,7 +1265,7 @@ "hashes": [ { "alg": "SHA-256", - "content": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "content": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "licenses": [ @@ -1317,7 +1317,7 @@ }, { "name": "jinja2cpp:file", - "value": "src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "value": "src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" }, { "name": "jinja2cpp:generator", @@ -2093,6 +2093,62 @@ } ] }, + { + "type": "application", + "bom-ref": "libabigail", + "name": "libabigail (abigail-tools)", + "version": "2.4", + "supplier": { + "name": "Red Hat / sourceware.org" + }, + "scope": "excluded", + "licenses": [ + { + "expression": "Apache-2.0 WITH LLVM-exception" + } + ], + "purl": "pkg:generic/libabigail@2.4", + "externalReferences": [ + { + "type": "website", + "url": "https://sourceware.org/libabigail/" + } + ], + "properties": [ + { + "name": "jinja2cpp:role", + "value": "tooling" + }, + { + "name": "jinja2cpp:phases", + "value": "ci" + }, + { + "name": "jinja2cpp:exposure", + "value": "none" + }, + { + "name": "jinja2cpp:delivery", + "value": "system" + }, + { + "name": "jinja2cpp:selected-by", + "value": "linux-build CI abi rows (apt abigail-tools, Ubuntu 24.04)" + }, + { + "name": "jinja2cpp:used-for", + "value": "ABI dump and comparison of the shared library (abidw, abidiff)" + }, + { + "name": "jinja2cpp:why", + "value": "SOVERSION policy gate (docs/tasks/0076)" + }, + { + "name": "jinja2cpp:lighten", + "value": "Not part of any build" + } + ] + }, { "type": "application", "bom-ref": "cmake", @@ -2710,6 +2766,7 @@ "clang-tidy", "pyyaml", "clang-format", + "libabigail", "cmake", "conan", "lars-bilke-codecoverage", diff --git a/sbom/jinja2cpp-full.spdx.json b/sbom/jinja2cpp-full.spdx.json index 1af5aa90a..83ec1c886 100644 --- a/sbom/jinja2cpp-full.spdx.json +++ b/sbom/jinja2cpp-full.spdx.json @@ -427,7 +427,7 @@ } ], "primaryPackagePurpose": "SOURCE", - "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" }, { "SPDXID": "SPDXRef-Package-unicode-data", @@ -448,7 +448,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc; file: src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b; file: src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d; file: src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0; file: src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" }, { "SPDXID": "SPDXRef-Package-html-entities", @@ -460,7 +460,7 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "checksumValue": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "homepage": "https://docs.python.org/3.12/library/html.entities.html", @@ -475,7 +475,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e; generator: scripts/gen_html_entities.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6; generator: scripts/gen_html_entities.py" }, { "SPDXID": "SPDXRef-Package-googletest", @@ -748,6 +748,27 @@ "primaryPackagePurpose": "APPLICATION", "comment": "role: tooling; phases: dev; exposure: none; delivery: system; selected-by: format-check CI job (apt clang-format-18); used-for: Formatting gate; why: Code style; lighten: Not part of any build" }, + { + "SPDXID": "SPDXRef-Package-libabigail", + "name": "libabigail (abigail-tools)", + "versionInfo": "2.4", + "supplier": "Organization: Red Hat / sourceware.org", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "homepage": "https://sourceware.org/libabigail/", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Apache-2.0 WITH LLVM-exception", + "copyrightText": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libabigail@2.4" + } + ], + "primaryPackagePurpose": "APPLICATION", + "comment": "role: tooling; phases: ci; exposure: none; delivery: system; selected-by: linux-build CI abi rows (apt abigail-tools, Ubuntu 24.04); used-for: ABI dump and comparison of the shared library (abidw, abidiff); why: SOVERSION policy gate (docs/tasks/0076); lighten: Not part of any build" + }, { "SPDXID": "SPDXRef-Package-cmake", "name": "CMake", @@ -1130,6 +1151,11 @@ "relationshipType": "DEV_TOOL_OF", "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" }, + { + "spdxElementId": "SPDXRef-Package-libabigail", + "relationshipType": "DEV_TOOL_OF", + "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" + }, { "spdxElementId": "SPDXRef-Package-cmake", "relationshipType": "BUILD_TOOL_OF", @@ -1181,5 +1207,5 @@ "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" } ], - "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-full-6aee0b571f4d138f" + "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-full-5176e9284b3785f5" } diff --git a/sbom/jinja2cpp.cdx.json b/sbom/jinja2cpp.cdx.json index 258ac8517..9e0d616db 100644 --- a/sbom/jinja2cpp.cdx.json +++ b/sbom/jinja2cpp.cdx.json @@ -1,7 +1,7 @@ { "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:11349e2e-8f15-5505-bdfe-c51fdcc762fb", + "serialNumber": "urn:uuid:96e351fa-fe32-5440-b764-2411a152ef20", "version": 1, "metadata": { "timestamp": "2026-10-18T00:00:00Z", @@ -1163,11 +1163,11 @@ }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3" + "value": "include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a" }, { "name": "jinja2cpp:file", - "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "value": "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" } ] }, @@ -1229,15 +1229,15 @@ }, { "name": "jinja2cpp:file", - "value": "src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc" + "value": "src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d" }, { "name": "jinja2cpp:file", - "value": "src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b" + "value": "src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0" }, { "name": "jinja2cpp:file", - "value": "src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3" + "value": "src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243" }, { "name": "jinja2cpp:generator", @@ -1265,7 +1265,7 @@ "hashes": [ { "alg": "SHA-256", - "content": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "content": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "licenses": [ @@ -1317,7 +1317,7 @@ }, { "name": "jinja2cpp:file", - "value": "src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "value": "src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" }, { "name": "jinja2cpp:generator", diff --git a/sbom/jinja2cpp.spdx.json b/sbom/jinja2cpp.spdx.json index 25e53833f..405f03ff0 100644 --- a/sbom/jinja2cpp.spdx.json +++ b/sbom/jinja2cpp.spdx.json @@ -427,7 +427,7 @@ } ], "primaryPackagePurpose": "SOURCE", - "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:b00dba6183edea9532030a373956ecab3638b6c5cfe61f40cbe862fba701dce3; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:eb0e46a2c95e693be7a192c1a992ece6e6fb900235ac44620f220bd646b2e372" + "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" }, { "SPDXID": "SPDXRef-Package-unicode-data", @@ -448,7 +448,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:bb7537033f18a1847f7cdb89bb19bcdee61f8afe6fd0d24891ab6967bccb38dc; file: src/unicode_tables.h sha256:5d3160113753deb6394ff66d867522a1c0303da0bfcc83bafaa027cfbbe3b96b; file: src/unicode_case.h sha256:5e7881179bb8cedcc76abef282540138f317cda02286077ced320d06640ce3c3; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Code-point tables for str.isprintable/isalpha/..., upper/lower/title/casefold, int() digit handling; why: Python-identical string predicates and case mapping without ICU; lighten: Data only, no dependency at build time (Python is needed only to regenerate). Acquisition decision is still open in docs/tasks/0187 and 0186; file: src/unicode_printable.h sha256:7cc795a76469cb82072a59c4f19bf92c231668a28497236e9f90f68eb7887f4d; file: src/unicode_tables.h sha256:67feccba784328f3fa4f8ea65d5eb03ab0a118532df178c209777a72d38437c0; file: src/unicode_case.h sha256:6e6bc5f2a2b505fa82477a2f88c0f8a77e2d2adc2e7e136e869bae9a15fe2243; generator: scripts/gen_unicode_printable.py; generator: scripts/gen_unicode_tables.py; generator: scripts/gen_unicode_case.py" }, { "SPDXID": "SPDXRef-Package-html-entities", @@ -460,7 +460,7 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e" + "checksumValue": "faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6" } ], "homepage": "https://docs.python.org/3.12/library/html.entities.html", @@ -475,7 +475,7 @@ } ], "primaryPackagePurpose": "FILE", - "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:83c2e14ba7b52dfd5befbba8f3bca85e0f8bbaf5696cc647c8372602ebc5ed1e; generator: scripts/gen_html_entities.py" + "comment": "role: mandatory; phases: build; exposure: internal; delivery: generated; selected-by: always; used-for: Entity table for MarkupSafe-compatible unescape/striptags; why: Same entity set as Python's html.unescape; lighten: Data only; keep; file: src/html_entities.h sha256:faabd9fba5045264f159834f0f6d9eec1a281bb3349c7315231b6ac56d5f4ea6; generator: scripts/gen_html_entities.py" } ], "relationships": [ @@ -580,5 +580,5 @@ "relatedSpdxElement": "SPDXRef-Package-html-entities" } ], - "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-library-5b4dd302002fa2d1" + "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-library-289c5f2348cb91b6" } diff --git a/scripts/abi_report.sh b/scripts/abi_report.sh new file mode 100755 index 000000000..5064717dd --- /dev/null +++ b/scripts/abi_report.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Compares the library's ABI with the previous one for one C++ standard (docs/tasks/0076). +# scripts/abi_report.sh # after abidw wrote .abi/libjinja2cpp.so.2.cxx.abi +# Run by the abidiff steps of .github/workflows/linux-build.yml. +# +# Until the 2.0 RC no baseline is committed and the dump of the last master build +# (.abi-base/, restored from the Actions cache) is the reference: the report shows what a +# pull request changes and never fails. From the RC on, abi/libjinja2cpp.so..cxx.abi +# is the reference, and an incompatible change (abidiff bit 8) fails unless the library's +# SONAME differs from the baseline's, that is unless SOVERSION was bumped (in which case the +# pull request also commits the new dump as the baseline). +set -uo pipefail + +std=${1:?usage: abi_report.sh } +current=$(ls .abi/libjinja2cpp.so.*.cxx"$std".abi 2>/dev/null | head -n 1) +baseline=$(ls abi/libjinja2cpp.so.*.cxx"$std".abi 2>/dev/null | head -n 1) +mode=gate +if [ -z "$baseline" ]; then + mode=report + baseline=$(ls .abi-base/libjinja2cpp.so.*.cxx"$std".abi 2>/dev/null | head -n 1) +fi +summary=${GITHUB_STEP_SUMMARY:-/dev/null} + +if [ -z "$current" ]; then + echo "::error::no ABI dump for C++$std in .abi/" + exit 1 +fi +if [ -z "$baseline" ]; then + echo "No previous ABI dump for C++$std yet: nothing to compare." | tee -a "$summary" + exit 0 +fi + +abidiff --suppressions abi/suppressions.ini --no-show-locs "$baseline" "$current" > .abi/abidiff.txt 2>&1 +rc=$? +cat .abi/abidiff.txt + +soname() { grep -o -m 1 "soname='[^']*'" "$1" | cut -d"'" -f2; } +verdict="no ABI change" +if (( rc & 3 )); then + verdict="abidiff failed (exit $rc)" +elif (( rc & 8 )); then + verdict="incompatible ABI change" +elif (( rc & 4 )); then + verdict="compatible ABI change (additions)" +fi + +{ + echo "### ABI, C++$std: $verdict" + echo + echo "Compared with \`$baseline\` ($mode mode)." + if (( rc != 0 )); then + echo + echo '
abidiff report' + echo + echo '```' + head -n 400 .abi/abidiff.txt + echo '```' + echo '
' + fi +} >> "$summary" + +if [ "$mode" = report ]; then + if (( rc & 3 )); then + echo "::warning::$verdict; report mode does not fail" + elif (( rc & 8 )); then + echo "::notice::C++$std: $verdict against the last master build (report only until the 2.0 RC)" + fi + exit 0 +fi +if (( rc & 3 )); then + echo "::error::$verdict" + exit 1 +fi +if (( rc & 8 )); then + if [ "$(soname "$baseline")" != "$(soname "$current")" ]; then + echo "::notice::C++$std: $verdict with SONAME $(soname "$current") (was $(soname "$baseline")); commit the new dump as the baseline" + exit 0 + fi + echo "::error::C++$std: $verdict without a SOVERSION bump. Bump SOVERSION in CMakeLists.txt and commit the job's dump (artifact abi-cxx$std) as abi/, or keep the ABI (docs/tasks/0076)." + exit 1 +fi +exit 0 diff --git a/test/abi_symbols.cmake b/test/abi_symbols.cmake new file mode 100644 index 000000000..71bdc935c --- /dev/null +++ b/test/abi_symbols.cmake @@ -0,0 +1,48 @@ +# Fails when the library defines a jinja2 symbol outside the inline ABI namespace (docs/tasks/0076). +# cmake -DNM= -DLIBRARY= -P test/abi_symbols.cmake +# Every mangled jinja2:: name must start jinja2::v2:: (`6jinja22v2`), and the vendored +# polymorphic/indirect types must not be left in a global `xyz`, where a user's own copy of +# the upstream headers would collide with them. scripts/abi_namespace.py keeps the sources +# in that shape; this checks what the compiler actually emitted. +foreach (_var NM LIBRARY) + if (NOT ${_var}) + message(FATAL_ERROR "${_var} is required") + endif () +endforeach () +# Defined symbols only: a static library repeats every reference in each object +execute_process(COMMAND "${NM}" --defined-only "${LIBRARY}" OUTPUT_VARIABLE _symbols ERROR_VARIABLE _err RESULT_VARIABLE _rc) +if (NOT _rc EQUAL 0) + execute_process(COMMAND "${NM}" "${LIBRARY}" OUTPUT_VARIABLE _symbols ERROR_VARIABLE _err RESULT_VARIABLE _rc) +endif () +if (NOT _rc EQUAL 0) + message(FATAL_ERROR "${NM} ${LIBRARY} failed (${_rc}): ${_err}") +endif () +# Mach-O prefixes every symbol with an extra underscore; the match starts after it +string(REGEX MATCHALL "_Z[^ \n]*" _syms "${_symbols}") +set(_tagged ${_syms}) +list(FILTER _tagged INCLUDE REGEX "(^|[^0-9])6jinja22v2") +list(LENGTH _tagged _tagged) +set(_rest ${_syms}) +list(TRANSFORM _rest REPLACE "(^|[^0-9])6jinja22v2" "\\1") +list(FILTER _rest INCLUDE REGEX "(^|[^0-9])6jinja2|^_ZN?K?3xyz") +# report the symbols as nm printed them +set(_bad "") +if (_rest) + foreach (_sym IN LISTS _syms) + string(REGEX REPLACE "(^|[^0-9])6jinja22v2" "\\1" _stripped "${_sym}") + if (_stripped MATCHES "(^|[^0-9])6jinja2|^_ZN?K?3xyz") + list(APPEND _bad "${_sym}") + endif () + endforeach () +endif () +list(REMOVE_DUPLICATES _bad) +list(LENGTH _bad _nbad) +if (_nbad GREATER 0) + list(SUBLIST _bad 0 20 _shown) + string(REPLACE ";" "\n " _shown "${_shown}") + message(FATAL_ERROR "${_nbad} symbols outside jinja2::v2 in ${LIBRARY}, run scripts/abi_namespace.py:\n ${_shown}") +endif () +if (_tagged EQUAL 0) + message(FATAL_ERROR "no jinja2::v2 symbols in ${LIBRARY}: is ${NM} reading the right file?") +endif () +message(STATUS "${_tagged} jinja2::v2 symbols, none outside it") diff --git a/test/package/CMakeLists.txt b/test/package/CMakeLists.txt index 0a78725e6..e455b2538 100644 --- a/test/package/CMakeLists.txt +++ b/test/package/CMakeLists.txt @@ -18,6 +18,21 @@ enable_testing() add_test(NAME consumer COMMAND consumer ${CMAKE_CURRENT_BINARY_DIR}/templates) add_test(NAME consumer_compat COMMAND consumer_compat ${CMAKE_CURRENT_BINARY_DIR}/templates) +# A program built against 1.x headers must fail to link with this library, not misbehave: +# the inline namespace v2 is in every mangled name (docs/tasks/0076). abi_probe_v2 is the +# positive control; abi_probe_v1 is built only by the test, which expects the link error. +add_executable(abi_probe_v2 abi_probe.cpp) +target_compile_definitions(abi_probe_v2 PRIVATE PROBE_V2) +target_link_libraries(abi_probe_v2 PRIVATE jinja2cpp::jinja2cpp) +add_test(NAME abi_probe_v2 COMMAND abi_probe_v2) +add_executable(abi_probe_v1 EXCLUDE_FROM_ALL abi_probe.cpp) +target_link_libraries(abi_probe_v1 PRIVATE jinja2cpp::jinja2cpp) +add_test(NAME abi_probe_v1_does_not_link + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target abi_probe_v1 --config $) +# GNU ld, lld, Apple ld and MSVC link; a compile error or a successful link does not match +set_tests_properties(abi_probe_v1_does_not_link PROPERTIES PASS_REGULAR_EXPRESSION + "undefined reference to [`']?jinja2::operator==|undefined symbol: jinja2::operator==|Undefined symbols for architecture|unresolved external symbol") + # check_package.cmake points PKG_CONFIG_PATH at the install and turns this on. pkg-config # lists the private dependencies of a static build only in the internal deps mode, where # it knows them. diff --git a/test/package/abi_probe.cpp b/test/package/abi_probe.cpp new file mode 100644 index 000000000..4747abe8b --- /dev/null +++ b/test/package/abi_probe.cpp @@ -0,0 +1,28 @@ +// Negative link check for the inline ABI namespace (docs/tasks/0076). No Jinja2C++ header is +// included: the declaration below is what a program compiled against the 1.x headers refers +// to. Built with PROBE_V2 it names jinja2::v2::operator== and must link and run; without it, +// it names the 1.x jinja2::operator==, which the 2.x library must not define, so the link +// has to fail (test/package/CMakeLists.txt). +#ifdef PROBE_V2 +namespace jinja2 +{ +inline namespace v2 +{ +class Value; +bool operator==(const Value& lhs, const Value& rhs); +} // namespace v2 +} // namespace jinja2 +#else +namespace jinja2 +{ +class Value; +bool operator==(const Value& lhs, const Value& rhs); +} // namespace jinja2 +#endif + +int main() +{ + // volatile keeps the reference to the library symbol in the object file + bool (*volatile compare)(const jinja2::Value&, const jinja2::Value&) = &jinja2::operator==; + return compare ? 0 : 1; +} diff --git a/test/package/check_package.cmake b/test/package/check_package.cmake index 4739b5d07..c3d320c5f 100644 --- a/test/package/check_package.cmake +++ b/test/package/check_package.cmake @@ -51,6 +51,28 @@ foreach (_req "${_major}.0;TRUE" "${_next}.0;FALSE" "${_prev}.0;FALSE") message(FATAL_ERROR "version file: ${PACKAGE_FIND_VERSION} compatible=${PACKAGE_VERSION_COMPATIBLE}, want ${_want}") endif () endforeach () +# SOVERSION is the major version (docs/tasks/0076): the loader refuses a shared library of +# another major, which is what protects shared consumers across an ABI break +if (EXPECT_SHARED AND NOT CMAKE_HOST_WIN32) + if (CMAKE_HOST_APPLE) + set(_soname "libjinja2cpp.${_major}.dylib") + else () + set(_soname "libjinja2cpp.so.${_major}") + endif () + file(GLOB _solib "${PREFIX}/lib/${_soname}" "${PREFIX}/lib64/${_soname}" "${PREFIX}/lib/*/${_soname}") + if (NOT _solib) + file(GLOB_RECURSE _installed RELATIVE "${PREFIX}" "${PREFIX}/*jinja2cpp*") + message(FATAL_ERROR "${_soname} is not installed; found: ${_installed}") + endif () + list(GET _solib 0 _solib) + find_program(_readelf readelf) + if (_readelf AND NOT CMAKE_HOST_APPLE) + execute_process(COMMAND "${_readelf}" -d "${_solib}" OUTPUT_VARIABLE _dynamic) + if (NOT _dynamic MATCHES "SONAME\\)? *Library soname: \\[${_soname}\\]") + message(FATAL_ERROR "SONAME of ${_solib} is not ${_soname}:\n${_dynamic}") + endif () + endif () +endif () set(ENV{PKG_CONFIG_PATH} "") file(GLOB_RECURSE _pcdirs LIST_DIRECTORIES true "${PREFIX}/*/pkgconfig") string(REPLACE ";" ":" _pcpath "${_pcdirs}") diff --git a/test/package/main.cpp b/test/package/main.cpp index 63cac50d2..1a7196d5c 100644 --- a/test/package/main.cpp +++ b/test/package/main.cpp @@ -12,8 +12,9 @@ static_assert(nsel_USES_STD_EXPECTED == 0, "Result must be expected-lite's ow #if defined(EXPECT_SHARED) != defined(JINJA2CPP_LINK_AS_SHARED) #error "JINJA2CPP_LINK_AS_SHARED does not match the installed library's link type" #endif -// The generated version.h is installed with the headers (docs/tasks/0069) -static_assert(JINJA2CPP_VERSION >= 10302, "jinja2cpp/version.h is stale or missing"); +// The generated version.h is installed with the headers (docs/tasks/0069), and its major +// version is the one in the inline ABI namespace v2 (docs/tasks/0076) +static_assert(JINJA2CPP_VERSION_MAJOR == 2, "jinja2cpp/version.h is stale or missing, or the ABI namespace tag needs a new major"); namespace { diff --git a/test/template_api_test.cpp b/test/template_api_test.cpp index 666a2bf9e..af1c4ccb6 100644 --- a/test/template_api_test.cpp +++ b/test/template_api_test.cpp @@ -338,7 +338,8 @@ TEST(TemplateApiTest, ConcurrentRenderOfOneTemplate) TEST(TemplateApiTest, VersionMacrosMatchProjectVersion) { static_assert(JINJA2CPP_VERSION == (JINJA2CPP_VERSION_MAJOR * 10000) + (JINJA2CPP_VERSION_MINOR * 100) + JINJA2CPP_VERSION_PATCH); - static_assert(JINJA2CPP_VERSION >= 10302); + // the inline ABI namespace is v2 (docs/tasks/0076); a new major retags it with scripts/abi_namespace.py + static_assert(JINJA2CPP_VERSION_MAJOR == 2); const auto expected = std::to_string(JINJA2CPP_VERSION_MAJOR) + "." + std::to_string(JINJA2CPP_VERSION_MINOR) + "." + std::to_string(JINJA2CPP_VERSION_PATCH); EXPECT_EQ(expected, JINJA2CPP_VERSION_STRING); } From 577860d33861a092019f127e0f79b9d81e0d7d6b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 01:53:23 +0000 Subject: [PATCH 13/18] 0076 PR-2 verifier fixes: generators emit v2, abidiff gates layout changes - gen_html_entities/unicode_case/unicode_tables/unicode_printable emit the inline namespace v2 form, so their --check matches the wrapped headers - abi_report.sh: any change left after --no-added-syms is a break (abidiff reports a changed layout with bit 4, not 8); artifact uploaded on failure - abi_namespace.py rejects spellings it does not rewrite (indented, one-line) - docs/dependencies.md and SBOM name jinja2::v2::detail::xyz Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Ua8ABM1TRsAvso2sWGHFSw --- .github/workflows/linux-build.yml | 3 ++- docs/dependencies.md | 2 +- docs/tasks/0076-abi-namespace-and-headers.md | 6 ++++- docs/tasks/0285-abi-baselines-at-rc.md | 7 +++++- sbom/components.json | 4 ++-- sbom/jinja2cpp-full.cdx.json | 8 +++---- sbom/jinja2cpp-full.spdx.json | 4 ++-- sbom/jinja2cpp.cdx.json | 8 +++---- sbom/jinja2cpp.spdx.json | 4 ++-- scripts/abi_namespace.py | 6 +++++ scripts/abi_report.sh | 25 +++++++++++++------- scripts/gen_html_entities.py | 4 ++-- scripts/gen_unicode_case.py | 4 ++-- scripts/gen_unicode_printable.py | 4 ++-- scripts/gen_unicode_tables.py | 4 ++-- 15 files changed, 58 insertions(+), 35 deletions(-) diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index a75d72278..504d73be8 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -138,7 +138,8 @@ jobs: path: .abi-base key: abi-cxx${{ matrix.std }}-${{ github.sha }} - - if: matrix.abi + # also when the report fails: its message points at this dump + - if: ${{ !cancelled() && matrix.abi }} uses: actions/upload-artifact@v4 with: name: abi-cxx${{ matrix.std }} diff --git a/docs/dependencies.md b/docs/dependencies.md index fc5f47a53..2252efab7 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -45,7 +45,7 @@ source and committed), `system` (found on the machine), `ci` (GitHub Actions). | Boost | 1.92.0 | BSL-1.0 | mandatory | link | fetched | the nine libraries below | Yes, piece by piece (docs/tasks/0230) | | robin-hood-hashing | 3.11.5, modified | MIT | mandatory | internal | vendored | flat hash map for values and scopes, faster than `std::unordered_map` here | Keep the idea; upstream is archived (docs/tasks/0231) | | lexertk | unrecorded, modified | MIT | mandatory | internal | vendored | expression tokenizer | Keep; effectively forked code | -| value_types | 57f2e7a, modified | MIT | mandatory | public-api | vendored | `polymorphic`/`indirect` in `jinja2::detail::xyz`, behind `jinja2::types` (Value, GenericList) | Keep; `std::` at C++26 | +| value_types | 57f2e7a, modified | MIT | mandatory | public-api | vendored | `polymorphic`/`indirect` in `jinja2::v2::detail::xyz`, behind `jinja2::types` (Value, GenericList) | Keep; `std::` at C++26 | | Unicode tables | UCD 15.0.0 via Python 3.12 | Unicode-3.0 | mandatory | internal | generated | Python-identical `isprintable`, case mapping, digits without ICU | Data only; acquisition decision open in 0187/0186 | | HTML entities | Python 3.12 `html.entities` | PSF-2.0 | mandatory | internal | generated | MarkupSafe-compatible unescape | Data only; keep | | Boost.JSON | 1.92.0 | BSL-1.0 | alternative (default) | public-optional | fetched | parses `{% meta %}` JSON for `GetMetadata` | Yes: the heaviest backend for a niche feature | diff --git a/docs/tasks/0076-abi-namespace-and-headers.md b/docs/tasks/0076-abi-namespace-and-headers.md index f0ae981e9..6ad4b1fbf 100644 --- a/docs/tasks/0076-abi-namespace-and-headers.md +++ b/docs/tasks/0076-abi-namespace-and-headers.md @@ -47,4 +47,8 @@ major 2; README "Compatibility and versioning". The abidiff job runs on one shar RelWithDebInfo row per standard and reports against the last master build; committing the baselines at the RC is 0285. Deliberate divergence from the plan: `jinja2::v2::detail` is not suppressed in abidiff (exported detail functions are called from inline headers), and -the C++20 abi row is gcc-12, not gcc-14, so that the Release rows keep every pair. +the C++20 abi row is gcc-12, not gcc-14, so that the Release rows keep every pair; the C++17 +abi row stays clang-18 (the plan preferred GCC, but the only C++17 shared row is clang-18). +The gate counts any change left after `--no-added-syms` as a break, since abidiff reports +a changed layout (a new `Settings` field) with bit 4, not bit 8. The four Unicode/HTML +generators in scripts/ emit the `v2` form, so their `--check` stays green. diff --git a/docs/tasks/0285-abi-baselines-at-rc.md b/docs/tasks/0285-abi-baselines-at-rc.md index 82cb6ada8..b842a200d 100644 --- a/docs/tasks/0285-abi-baselines-at-rc.md +++ b/docs/tasks/0285-abi-baselines-at-rc.md @@ -20,7 +20,12 @@ incompatible change unless SOVERSION was bumped. that, read one report and review `abi/suppressions.ini`: it suppresses only the vendored value_types and robin_hood/lexertk names; `jinja2::v2::detail` stays checked because inline header code calls exported detail functions. Add suppressions only for noise that -is shown not to reach users. +is shown not to reach users. Two things to settle first: a RelWithDebInfo dump is about 31 MB +(1.2 MB gzipped), so commit the baselines compressed or trimmed and teach +`scripts/abi_report.sh` to read them; and the dump includes weak std and Boost symbols the +library exports (`boost::system` among them), so a Boost pin bump may read as a break +unless those are suppressed or hidden. `scripts/**` is in linux-build's `paths-ignore`, so +a PR that touches only `abi_report.sh` runs no abi row: check such a change by hand. **Done when.** The baselines are on master, a deliberate test change to an exported layout makes the abi rows red, and docs/release-2.0.md lists the step in its RC procedure. diff --git a/sbom/components.json b/sbom/components.json index c2012833a..1d94569ac 100644 --- a/sbom/components.json +++ b/sbom/components.json @@ -278,12 +278,12 @@ "purl": "pkg:github/jbcoe/value_types@57f2e7a", "delivery": "vendored", "files": ["include/jinja2cpp/polymorphic_value/indirect_cxx14.h", "include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h"], - "modified": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T", + "modified": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T", "role": "mandatory", "phases": ["build", "consumer-build"], "exposure": "public-api", "selected_by": "always", - "used_for": "polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap", + "used_for": "polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap", "why": "Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26", "lighten": "Keep (header-only, installed). Switch to std:: at a C++26 floor" }, diff --git a/sbom/jinja2cpp-full.cdx.json b/sbom/jinja2cpp-full.cdx.json index 0af193245..5f9660898 100644 --- a/sbom/jinja2cpp-full.cdx.json +++ b/sbom/jinja2cpp-full.cdx.json @@ -1,7 +1,7 @@ { "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:05ea2675-8382-5679-ad05-ef9a8b7bf331", + "serialNumber": "urn:uuid:a7ddc095-8bb8-5764-b9ea-9b60eb28ef13", "version": 1, "metadata": { "timestamp": "2026-10-18T00:00:00Z", @@ -1122,7 +1122,7 @@ } ], "pedigree": { - "notes": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "notes": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, "properties": [ { @@ -1147,7 +1147,7 @@ }, { "name": "jinja2cpp:used-for", - "value": "polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" + "value": "polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" }, { "name": "jinja2cpp:why", @@ -1159,7 +1159,7 @@ }, { "name": "jinja2cpp:modified", - "value": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "value": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, { "name": "jinja2cpp:file", diff --git a/sbom/jinja2cpp-full.spdx.json b/sbom/jinja2cpp-full.spdx.json index 83ec1c886..9bf8e4c0c 100644 --- a/sbom/jinja2cpp-full.spdx.json +++ b/sbom/jinja2cpp-full.spdx.json @@ -427,7 +427,7 @@ } ], "primaryPackagePurpose": "SOURCE", - "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" + "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" }, { "SPDXID": "SPDXRef-Package-unicode-data", @@ -1207,5 +1207,5 @@ "relatedSpdxElement": "SPDXRef-Package-jinja2cpp" } ], - "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-full-5176e9284b3785f5" + "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-full-45343781f625a58f" } diff --git a/sbom/jinja2cpp.cdx.json b/sbom/jinja2cpp.cdx.json index 9e0d616db..e35ee8cd0 100644 --- a/sbom/jinja2cpp.cdx.json +++ b/sbom/jinja2cpp.cdx.json @@ -1,7 +1,7 @@ { "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": "urn:uuid:96e351fa-fe32-5440-b764-2411a152ef20", + "serialNumber": "urn:uuid:6517a686-4ed3-5d9a-9ee7-94f64aa14c22", "version": 1, "metadata": { "timestamp": "2026-10-18T00:00:00Z", @@ -1122,7 +1122,7 @@ } ], "pedigree": { - "notes": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "notes": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, "properties": [ { @@ -1147,7 +1147,7 @@ }, { "name": "jinja2cpp:used-for", - "value": "polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" + "value": "polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap" }, { "name": "jinja2cpp:why", @@ -1159,7 +1159,7 @@ }, { "name": "jinja2cpp:modified", - "value": "both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" + "value": "both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T" }, { "name": "jinja2cpp:file", diff --git a/sbom/jinja2cpp.spdx.json b/sbom/jinja2cpp.spdx.json index 405f03ff0..fe2683991 100644 --- a/sbom/jinja2cpp.spdx.json +++ b/sbom/jinja2cpp.spdx.json @@ -427,7 +427,7 @@ } ], "primaryPackagePurpose": "SOURCE", - "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" + "comment": "role: mandatory; phases: build,consumer-build; exposure: public-api; delivery: vendored; selected-by: always; used-for: polymorphic and indirect (jinja2::v2::detail::xyz) behind jinja2::types and jinja2::detail (value_ptr.h), used by Value, GenericList and GenericMap; why: Value-semantic owning pointers for the public value model; standardised as std::indirect/std::polymorphic in C++26; lighten: Keep (header-only, installed). Switch to std:: at a C++26 floor; modified: both at upstream 57f2e7a, moved to namespace jinja2::v2::detail::xyz with JINJA2CPP_XYZ_ macros (0076); indirect: Clang incomplete-type fix (0edc34d), tag tested first (0072); polymorphic: default constructor constrained on the allocator for MSVC std::variant (0071), cb_ initialised (0089), in-place initializer_list constructor creates U, not T; file: include/jinja2cpp/polymorphic_value/indirect_cxx14.h sha256:14f328d0de7e26f047178a58d443b0316a2ecd69e1ec05007d6a3aeb23290d6a; file: include/jinja2cpp/polymorphic_value/polymorphic_cxx14.h sha256:3d46bb50b19a3f22284aa89866a9a521325456bf3e00f7245a42852ffb9a4db1" }, { "SPDXID": "SPDXRef-Package-unicode-data", @@ -580,5 +580,5 @@ "relatedSpdxElement": "SPDXRef-Package-html-entities" } ], - "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-library-289c5f2348cb91b6" + "documentNamespace": "https://github.com/jinja2cpp/Jinja2Cpp/sbom/jinja2cpp-2.0.0-library-751d894c115a1ae6" } diff --git a/scripts/abi_namespace.py b/scripts/abi_namespace.py index 9b6e3652b..b400f1c30 100755 --- a/scripts/abi_namespace.py +++ b/scripts/abi_namespace.py @@ -36,6 +36,7 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ROOTS = ("include", "src") OPENING = re.compile(r"^namespace jinja2(?P(?:::\w+)*)(?P\s*\{)?\s*$") +ANY_OPENING = re.compile(r"(?v\d+)(?P\s*\{)?\s*$") CLOSE_COMMENT = re.compile(r"^\}(?P\s*)//\s*namespace\s+(?P[\w:]+)\s*$") @@ -162,6 +163,11 @@ def rewrite(text): edit = (open_line + 1, open_line + 1, insert, close_line, ["}" + space + f"// namespace {TAG}", close_text]) break if edit is None: + # Any other spelling (indented, one-line `namespace jinja2 { ... }`) would stay + # outside v2 unnoticed: a free function in it becomes a second overload + for li, code in enumerate(mask_lines): + if ANY_OPENING.search(code) and not OPENING.match(code.rstrip()): + raise ValueError(f"line {li + 1}: write `namespace jinja2` alone at the start of a line") return "\n".join(lines) if edit == (): continue diff --git a/scripts/abi_report.sh b/scripts/abi_report.sh index 5064717dd..53569c6dc 100755 --- a/scripts/abi_report.sh +++ b/scripts/abi_report.sh @@ -6,9 +6,11 @@ # Until the 2.0 RC no baseline is committed and the dump of the last master build # (.abi-base/, restored from the Actions cache) is the reference: the report shows what a # pull request changes and never fails. From the RC on, abi/libjinja2cpp.so..cxx.abi -# is the reference, and an incompatible change (abidiff bit 8) fails unless the library's +# is the reference, and an incompatible change fails unless the library's # SONAME differs from the baseline's, that is unless SOVERSION was bumped (in which case the -# pull request also commits the new dump as the baseline). +# pull request also replaces the baseline with the new dump). An incompatible change is any +# change left once added symbols are ignored: removed or changed functions and variables, +# and changed layouts of the types they use. set -uo pipefail std=${1:?usage: abi_report.sh } @@ -33,12 +35,17 @@ fi abidiff --suppressions abi/suppressions.ini --no-show-locs "$baseline" "$current" > .abi/abidiff.txt 2>&1 rc=$? cat .abi/abidiff.txt +# Without the added symbols, any remaining change is a break: abidiff sets bit 8 only for +# removed symbols and vtable changes, and reports a changed layout (a new Settings field, a +# Value alternative, a widened member) with bit 4 alone +abidiff --suppressions abi/suppressions.ini --no-show-locs --no-added-syms "$baseline" "$current" > /dev/null 2>&1 +break_rc=$? soname() { grep -o -m 1 "soname='[^']*'" "$1" | cut -d"'" -f2; } verdict="no ABI change" -if (( rc & 3 )); then - verdict="abidiff failed (exit $rc)" -elif (( rc & 8 )); then +if (( (rc | break_rc) & 3 )); then + verdict="abidiff failed (exit $rc, $break_rc)" +elif (( break_rc & 12 )); then verdict="incompatible ABI change" elif (( rc & 4 )); then verdict="compatible ABI change (additions)" @@ -60,18 +67,18 @@ fi } >> "$summary" if [ "$mode" = report ]; then - if (( rc & 3 )); then + if (( (rc | break_rc) & 3 )); then echo "::warning::$verdict; report mode does not fail" - elif (( rc & 8 )); then + elif (( break_rc & 12 )); then echo "::notice::C++$std: $verdict against the last master build (report only until the 2.0 RC)" fi exit 0 fi -if (( rc & 3 )); then +if (( (rc | break_rc) & 3 )); then echo "::error::$verdict" exit 1 fi -if (( rc & 8 )); then +if (( break_rc & 12 )); then if [ "$(soname "$baseline")" != "$(soname "$current")" ]; then echo "::notice::C++$std: $verdict with SONAME $(soname "$current") (was $(soname "$baseline")); commit the new dump as the baseline" exit 0 diff --git a/scripts/gen_html_entities.py b/scripts/gen_html_entities.py index 713b747d0..8a5149bce 100644 --- a/scripts/gen_html_entities.py +++ b/scripts/gen_html_entities.py @@ -32,12 +32,12 @@ def main(): "// Generated by scripts/gen_html_entities.py from Python {}.{}'s html.entities.html5; do not edit.\n".format(*sys.version_info[:2]), "#ifndef JINJA2CPP_SRC_HTML_ENTITIES_H\n#define JINJA2CPP_SRC_HTML_ENTITIES_H\n", "#include \n#include \n#include \n#include \n", - "namespace jinja2::html\n{\n", + "namespace jinja2\n{\ninline namespace v2\n{\nnamespace html\n{\n", "// A named character reference and the one or two code points it stands for (0 when only one)\n" "struct HtmlEntity\n{\n const char* name;\n uint32_t first;\n uint32_t second;\n};\n", "// Sorted by name\nconstexpr HtmlEntity HtmlEntities[] = {\n" + "\n".join(rows) + "\n};\n", HELPER, - "} // namespace jinja2::html\n\n#endif // JINJA2CPP_SRC_HTML_ENTITIES_H\n", + "} // namespace html\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_HTML_ENTITIES_H\n", ] target = ROOT / "src" / "html_entities.h" text = "\n".join(parts) diff --git a/scripts/gen_unicode_case.py b/scripts/gen_unicode_case.py index 6cd74f592..572af3e6e 100644 --- a/scripts/gen_unicode_case.py +++ b/scripts/gen_unicode_case.py @@ -141,7 +141,7 @@ def main(): "// Generated by scripts/gen_unicode_case.py from Unicode {}; do not edit.\n".format(unicodedata.unidata_version), "#ifndef JINJA2CPP_SRC_UNICODE_CASE_H\n#define JINJA2CPP_SRC_UNICODE_CASE_H\n", '#include "unicode_tables.h"\n\n#include \n#include \n#include \n#include \n', - "namespace jinja2::unicode\n{\n", + "namespace jinja2\n{\ninline namespace v2\n{\nnamespace unicode\n{\n", "// cp + delta for every stride-th code point from first to last\n" "struct CaseRun\n{\n uint32_t first;\n uint32_t last;\n uint32_t stride;\n int32_t delta;\n};\n", "// A mapping to two or three code points; unused entries are 0\n" @@ -154,7 +154,7 @@ def main(): parts.append(range_table("Cased", "Cased and not Case_Ignorable", ranges(cased))) parts.append(range_table("CaseIgnorable", "Case_Ignorable (cased or not)", ranges(case_ignorable))) parts.append(HELPERS) - parts.append("} // namespace jinja2::unicode\n\n#endif // JINJA2CPP_SRC_UNICODE_CASE_H\n") + parts.append("} // namespace unicode\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_CASE_H\n") target = ROOT / "src" / "unicode_case.h" text = "\n".join(parts) # --check: CI (parity-expectations.yml) fails when the committed header is stale diff --git a/scripts/gen_unicode_printable.py b/scripts/gen_unicode_printable.py index 377c35d0e..6a9810d48 100644 --- a/scripts/gen_unicode_printable.py +++ b/scripts/gen_unicode_printable.py @@ -38,7 +38,7 @@ def generate(out): out.write("// Do not edit by hand.\n") out.write("#ifndef JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n#define JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n\n") out.write("#include \n#include \n#include \n\n") - out.write("namespace jinja2::detail\n{\n\n") + out.write("namespace jinja2\n{\ninline namespace v2\n{\nnamespace detail\n{\n\n") out.write("// True when Python's str.isprintable() holds for the code point, so repr() keeps it as is\n") out.write("inline bool IsPythonPrintable(uint32_t cp)\n{\n") out.write(" if (cp >= 0x20 && cp < 0x7f)\n {\n return true;\n }\n\n") @@ -51,7 +51,7 @@ def generate(out): " [](uint32_t val, const Range& r) { return val < r.first; });\n") out.write(" if (it == std::begin(nonPrintable))\n {\n return true;\n }\n") out.write(" --it;\n return cp > it->last;\n}\n\n") - out.write("} // namespace jinja2::detail\n\n#endif // JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n") + out.write("} // namespace detail\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_PRINTABLE_H\n") def main(): diff --git a/scripts/gen_unicode_tables.py b/scripts/gen_unicode_tables.py index 9ca02b799..dd71049f9 100755 --- a/scripts/gen_unicode_tables.py +++ b/scripts/gen_unicode_tables.py @@ -37,7 +37,7 @@ def main(): "// Generated by scripts/gen_unicode_tables.py from Unicode {}; do not edit.\n".format(unicodedata.unidata_version), "#ifndef JINJA2CPP_SRC_UNICODE_TABLES_H\n#define JINJA2CPP_SRC_UNICODE_TABLES_H\n", "#include \n#include \n#include \n", - "namespace jinja2\n{\nnamespace unicode\n{\n", + "namespace jinja2\n{\ninline namespace v2\n{\nnamespace unicode\n{\n", "struct CodePointRange\n{\n uint32_t first;\n uint32_t last;\n};\n", table("WordRanges", "Python \\w above ASCII: str.isalnum()", ranges(lambda c: c.isalnum())), table("DecimalRanges", "Python \\d above ASCII: str.isdecimal()", ranges(lambda c: c.isdecimal())), @@ -73,7 +73,7 @@ def main(): return InRanges(SpaceRanges, cp); } """, - "} // namespace unicode\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_TABLES_H\n", + "} // namespace unicode\n} // namespace v2\n} // namespace jinja2\n\n#endif // JINJA2CPP_SRC_UNICODE_TABLES_H\n", ] target = ROOT / "src" / "unicode_tables.h" text = "\n".join(parts) From 1c84b543d3d8af9003856ce84e40533e38c17ca8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 01:59:02 +0000 Subject: [PATCH 14/18] Tests: define detail specialisations by qualified name for MSVC (0076) MSVC does not implement CWG 2061: reopening namespace detail inside namespace jinja2 declares a new jinja2::detail beside jinja2::v2::detail. containers_api_test and node_arena_test now use the qualified form, and the README tells 1.x detail::Reflector users to do the same. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Ua8ABM1TRsAvso2sWGHFSw --- README.md | 7 ++++++- test/containers_api_test.cpp | 16 ++++++++-------- test/node_arena_test.cpp | 9 ++++----- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 716818956..037989d0d 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,12 @@ it in the default internal dependencies mode). The un-namespaced `jinja2cpp` tar `namespace jinja2 { class Value; }` declares a different, undefined class and makes later uses ambiguous. `` includes the whole API except the JSON bindings. If your code says `using namespace jinja2;` and has its own namespace-scope `v2`, qualify - one of them. + one of them. Specialise the extension points inside `namespace jinja2 { ... }` or with their + qualified name (`template<> struct jinja2::Reflector`). A 1.x-style + `detail::Reflector` specialisation must use the qualified form + (`template<> struct jinja2::detail::Reflector`): MSVC does not look through the + inline namespace when `namespace detail` is reopened inside `namespace jinja2`, and declares + a second `jinja2::detail` instead. - **ABI within 2.x.** The shared library's SONAME carries the ABI version (`libjinja2cpp.so.2`, `libjinja2cpp.2.dylib`). It is bumped by any release that changes the layout of a type you allocate (a `Settings` field, a `Value` alternative) or the signature diff --git a/test/containers_api_test.cpp b/test/containers_api_test.cpp index 5f3dda65c..46f24ad62 100644 --- a/test/containers_api_test.cpp +++ b/test/containers_api_test.cpp @@ -166,11 +166,13 @@ struct Reflector static Value CreateFromPtr(const Celsius* val) { return Create(*val); } }; -namespace detail -{ -// A 1.x-style specialisation keeps working +} // namespace jinja2 + +// A 1.x-style specialisation keeps working. It is written with the qualified name: MSVC does +// not implement CWG 2061, so `namespace jinja2 { namespace detail {` there declares a new +// jinja2::detail beside jinja2::v2::detail instead of reopening it (docs/tasks/0076, README) template<> -struct Reflector +struct jinja2::detail::Reflector { static Value Create(const LegacyReflected& val) { return Value(static_cast(val.value * 10)); } static Value CreateFromPtr(const LegacyReflected* val) { return Create(*val); } @@ -178,20 +180,18 @@ struct Reflector // 1.x code could delegate to the library's reflectors, and override one for a type the library also covers template<> -struct Reflector +struct jinja2::detail::Reflector { static Value Create(const LegacyDelegating& val) { return Reflector::Create(val.text + "!"); } static Value CreateFromPtr(const LegacyDelegating* val) { return Create(*val); } }; template<> -struct Reflector +struct jinja2::detail::Reflector { static Value Create(char16_t val) { return Value(std::string(1, static_cast(val))); } static Value CreateFromPtr(const char16_t* val) { return Create(*val); } }; -} // namespace detail -} // namespace jinja2 TEST(ContainersApiTest, ErrorCodeValuesAreStable) { diff --git a/test/node_arena_test.cpp b/test/node_arena_test.cpp index b204d31b9..9ec3f8cff 100644 --- a/test/node_arena_test.cpp +++ b/test/node_arena_test.cpp @@ -31,10 +31,10 @@ using namespace jinja2; -namespace jinja2::detail -{ -// Handles from raw offsets and the bytes of a sealed tree, to corrupt a tree on purpose -struct ArenaTestAccess +// Handles from raw offsets and the bytes of a sealed tree, to corrupt a tree on purpose. +// Defined through its qualified name: reopening `namespace jinja2::detail` here would make +// MSVC declare a new jinja2::detail beside jinja2::v2::detail (docs/tasks/0076) +struct jinja2::detail::ArenaTestAccess { template static NodeRef Ref(std::uint32_t offset) @@ -65,7 +65,6 @@ struct ArenaTestAccess return header.objects; } }; -} // namespace jinja2::detail namespace { From ede670728577f5b2a7dd4bb923b0307bb3a9fc24 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 03:46:52 +0000 Subject: [PATCH 15/18] 0076 PR-2 Codex fixes: SONAME from SOVERSION, baseline must match SONAME - check_package.cmake reads SOVERSION from CMakeLists.txt instead of using the project major, so a within-2.x SOVERSION bump installs and checks cleanly - abi_report.sh gate fails while the committed baseline's SONAME differs from the library's, so a SOVERSION bump cannot land without its new baseline Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Ua8ABM1TRsAvso2sWGHFSw --- scripts/abi_report.sh | 23 +++++++++++++---------- test/package/check_package.cmake | 14 ++++++++++---- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/scripts/abi_report.sh b/scripts/abi_report.sh index 53569c6dc..1ea53fa57 100755 --- a/scripts/abi_report.sh +++ b/scripts/abi_report.sh @@ -6,11 +6,12 @@ # Until the 2.0 RC no baseline is committed and the dump of the last master build # (.abi-base/, restored from the Actions cache) is the reference: the report shows what a # pull request changes and never fails. From the RC on, abi/libjinja2cpp.so..cxx.abi -# is the reference, and an incompatible change fails unless the library's -# SONAME differs from the baseline's, that is unless SOVERSION was bumped (in which case the -# pull request also replaces the baseline with the new dump). An incompatible change is any -# change left once added symbols are ignored: removed or changed functions and variables, -# and changed layouts of the types they use. +# is the reference and the gate fails +# - on an incompatible change: any change left once added symbols are ignored (removed or +# changed functions and variables, changed layouts of the types they use). The way out is +# to bump SOVERSION and replace the baseline with the new dump in the same pull request; +# - when the baseline's SONAME differs from the library's, so that a SOVERSION bump cannot +# land without its baseline. set -uo pipefail std=${1:?usage: abi_report.sh } @@ -78,12 +79,14 @@ if (( (rc | break_rc) & 3 )); then echo "::error::$verdict" exit 1 fi +# A SOVERSION bump is only complete with its baseline: until the committed dump carries the +# new SONAME, every later break would pass as "SONAME differs" +if [ "$(soname "$baseline")" != "$(soname "$current")" ]; then + echo "::error::C++$std: the library's SONAME is $(soname "$current") but $baseline is for $(soname "$baseline"). Replace the baseline with the job's dump (artifact abi-cxx$std) in the same pull request as the SOVERSION bump (docs/tasks/0076)." + exit 1 +fi if (( break_rc & 12 )); then - if [ "$(soname "$baseline")" != "$(soname "$current")" ]; then - echo "::notice::C++$std: $verdict with SONAME $(soname "$current") (was $(soname "$baseline")); commit the new dump as the baseline" - exit 0 - fi - echo "::error::C++$std: $verdict without a SOVERSION bump. Bump SOVERSION in CMakeLists.txt and commit the job's dump (artifact abi-cxx$std) as abi/, or keep the ABI (docs/tasks/0076)." + echo "::error::C++$std: $verdict without a SOVERSION bump. Bump SOVERSION in CMakeLists.txt and replace abi/ with the job's dump (artifact abi-cxx$std), or keep the ABI (docs/tasks/0076)." exit 1 fi exit 0 diff --git a/test/package/check_package.cmake b/test/package/check_package.cmake index c3d320c5f..5b2467329 100644 --- a/test/package/check_package.cmake +++ b/test/package/check_package.cmake @@ -51,13 +51,19 @@ foreach (_req "${_major}.0;TRUE" "${_next}.0;FALSE" "${_prev}.0;FALSE") message(FATAL_ERROR "version file: ${PACKAGE_FIND_VERSION} compatible=${PACKAGE_VERSION_COMPATIBLE}, want ${_want}") endif () endforeach () -# SOVERSION is the major version (docs/tasks/0076): the loader refuses a shared library of -# another major, which is what protects shared consumers across an ABI break +# The installed SONAME carries SOVERSION (docs/tasks/0076): the loader refuses a shared +# library of another ABI version, which is what protects shared consumers. SOVERSION tracks +# the ABI, not the major, so it is read from CMakeLists.txt rather than derived from _major. if (EXPECT_SHARED AND NOT CMAKE_HOST_WIN32) + file(STRINGS "${_src}/../../CMakeLists.txt" _soversion REGEX "^[ \t]*SOVERSION[ \t]+[0-9]+") + string(REGEX MATCH "[0-9]+" _soversion "${_soversion}") + if (NOT _soversion) + message(FATAL_ERROR "no SOVERSION found in ${_src}/../../CMakeLists.txt") + endif () if (CMAKE_HOST_APPLE) - set(_soname "libjinja2cpp.${_major}.dylib") + set(_soname "libjinja2cpp.${_soversion}.dylib") else () - set(_soname "libjinja2cpp.so.${_major}") + set(_soname "libjinja2cpp.so.${_soversion}") endif () file(GLOB _solib "${PREFIX}/lib/${_soname}" "${PREFIX}/lib64/${_soname}" "${PREFIX}/lib/*/${_soname}") if (NOT _solib) From ab8d075e9202c74e3b3a998553977314814d0a78 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 06:14:57 +0000 Subject: [PATCH 16/18] Merge train 28: dict keys of any hashable type (#594), ABI namespace jinja2::v2 (#604) --- docs/parity.md | 20 ++++++++++---------- docs/tasks/README.md | 5 +++-- src/serialize_filters.cpp | 11 ++++++++++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index 0021db676..b8e7f8f73 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -4,7 +4,7 @@ How far Jinja2C++ is from Python [Jinja2](https://jinja.palletsprojects.com/) 3. area, and which task in `docs/tasks/` closes each gap. Every statement marked with a case id is backed by the differential corpus in -`test/parity/` (1939 templates rendered by both engines, see +`test/parity/` (1995 templates rendered by both engines, see [test/parity/README.md](../test/parity/README.md)); `ctest -R parity` re-checks all of them. Statements in the last section (API level) are read from the headers and are not corpus-checked yet. @@ -20,24 +20,24 @@ Snapshot of `python3 test/parity/generate.py --report` (Jinja2 3.1.6, Oct 2026): |---|---|---|---|---|---|---|---|---|---| | autoescape | 124 | 122 | 1 | 1 | 0 | 0 | 0 | 0 | 0210, 0216 | | custom | 28 | 28 | 0 | 0 | 0 | 0 | 0 | 0 | | -| errors | 97 | 93 | 0 | 2 | 2 | 0 | 0 | 0 | 0036, 0213 | -| filters | 309 | 303 | 4 | 2 | 0 | 0 | 0 | 0 | 0101, 0178, 0203, 0215 | +| errors | 103 | 100 | 0 | 2 | 1 | 0 | 0 | 0 | 0213 | +| filters | 328 | 322 | 4 | 2 | 0 | 0 | 0 | 0 | 0101, 0178, 0203, 0215 | | fuzz | 32 | 32 | 0 | 0 | 0 | 0 | 0 | 0 | | -| globals | 74 | 73 | 1 | 0 | 0 | 0 | 0 | 0 | 0212 | +| globals | 78 | 77 | 1 | 0 | 0 | 0 | 0 | 0 | 0212 | | i18n | 54 | 54 | 0 | 0 | 0 | 0 | 0 | 0 | | -| literals | 68 | 65 | 2 | 1 | 0 | 0 | 0 | 0 | 0015, 0036, 0041 | +| literals | 75 | 73 | 1 | 1 | 0 | 0 | 0 | 0 | 0015, 0041 | | loader | 168 | 162 | 5 | 1 | 0 | 0 | 0 | 0 | 0165, 0211, 0274 | -| methods | 96 | 93 | 2 | 1 | 0 | 0 | 0 | 0 | 0049 | -| operators | 118 | 115 | 0 | 3 | 0 | 0 | 0 | 0 | 0015 | +| methods | 105 | 102 | 2 | 1 | 0 | 0 | 0 | 0 | 0049 | +| operators | 122 | 119 | 0 | 3 | 0 | 0 | 0 | 0 | 0015 | | options | 35 | 35 | 0 | 0 | 0 | 0 | 0 | 0 | | | output | 38 | 38 | 0 | 0 | 0 | 0 | 0 | 0 | | | sequences | 47 | 47 | 0 | 0 | 0 | 0 | 0 | 0 | | -| statements | 394 | 379 | 10 | 3 | 2 | 0 | 0 | 0 | 0156, 0165, 0209, 0214, 0219, 0220 | -| subscripts | 39 | 39 | 0 | 0 | 0 | 0 | 0 | 0 | | +| statements | 398 | 383 | 10 | 3 | 2 | 0 | 0 | 0 | 0156, 0165, 0209, 0214, 0219, 0220 | +| subscripts | 42 | 42 | 0 | 0 | 0 | 0 | 0 | 0 | | | tests | 58 | 57 | 0 | 1 | 0 | 0 | 0 | 0 | 0101 | | undefined | 106 | 106 | 0 | 0 | 0 | 0 | 0 | 0 | | | whitespace | 54 | 54 | 0 | 0 | 0 | 0 | 0 | 0 | | -| **total** | **1939** | **1895** | **25** | **15** | **4** | **0** | **0** | **0** | | +| **total** | **1995** | **1953** | **24** | **15** | **3** | **0** | **0** | **0** | | *output*: both render, text differs. *rejects*: C++ errors on a valid template. *accepts*: C++ renders a template Jinja2 rejects. *unsupported*: needs an Environment diff --git a/docs/tasks/README.md b/docs/tasks/README.md index cf529ee40..08c571747 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -107,7 +107,7 @@ files go under `touches` (0012 and 0024 each rewrite about a hundred rows). | [0033](0033-wide-string-parity.md) | Run the corpus through the wide-string API | parity | low | done | | [0034](0034-none-versus-undefined.md) | Tell `None` apart from undefined | parity | high | done | | [0035](0035-locale-independent-string-conversion.md) | Convert between narrow and wide strings without the C locale | robustness | medium | done | -| [0036](0036-non-string-mapping-keys.md) | Mapping keys that are not strings | parity | low | open | +| [0036](0036-non-string-mapping-keys.md) | Mapping keys that are not strings | parity | low | done | | [0037](0037-sequence-protocol-follow-ups.md) | Sequence protocol follow-ups from 0016 | parity | medium | done | | [0038](0038-lexical-scoping-for-macros.md) | Lexical scoping for macros | parity | medium | done | | [0039](0039-float-values-print-widened.md) | C++ `float` values print with their widened digits | parity | low | open | @@ -145,7 +145,7 @@ files go under `touches` (0012 and 0024 each rewrite about a hundred rows). | [0073](0073-template-api-2-0.md) | 2.0 API: `BasicTemplate` | release | medium | done | | [0074](0074-template-env-api-2-0.md) | 2.0 API: `TemplateEnv` pimpl and `Settings` | release | medium | done | | [0075](0075-containers-reflection-api-2-0.md) | 2.0 API: containers, reflection, errors | release | medium | done | -| [0076](0076-abi-namespace-and-headers.md) | 2.0: inline ABI namespace, header layout, version | release | high | in-progress | +| [0076](0076-abi-namespace-and-headers.md) | 2.0: inline ABI namespace, header layout, version | release | high | done | | [0077](0077-migration-script.md) | 2.0 migration script and notes | release | high | open | | [0078](0078-nix-toolchains.md) | Pinned toolchains from Nix for the tool and bleeding-edge rows | ci | low | open | | [0079](0079-msvc-runtime-shared-override.md) | Warn when `JINJA2CPP_MSVC_RUNTIME_TYPE` is overridden by a shared build | build | low | open | @@ -345,3 +345,4 @@ files go under `touches` (0012 and 0024 each rewrite about a hundred rows). | [0280](0280-urlize-markup-input.md) | urlize escapes Markup input | filters | low | open | | [0281](0281-attr-bound-method.md) | attr() of a method renders empty | filters | low | open | | [0284](0284-template-module.md) | `Template.module` / `make_module`: a template's exports as a map | api | low | open | +| [0285](0285-abi-baselines-at-rc.md) | Commit the ABI baselines at the 2.0 RC and turn the abidiff gate on | release | high | open | diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index 97ac2f990..d32df99a1 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -909,7 +909,16 @@ class YamlWriter // namespace(), cycler() and joiner() are objects, not data throw std::runtime_error("Object of type "s + objectClass + " is not YAML serializable"); } - const auto keys = map.GetKeys(); + // Keys other than strings (docs/tasks/0036) are not written as YAML mapping keys + std::vector keys; + for (const auto& entry : map.GetEntries()) + { + if (!IsStringValue(entry.key)) + { + throw std::runtime_error("Only string dictionary keys are YAML serializable"); + } + keys.push_back(AsString(entry.key)); + } if (keys.empty()) { StartNode(pos, indent, false); From 0b652be00c1efa2d0619ae3f57fc437a085e3e50 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 06:18:29 +0000 Subject: [PATCH 17/18] Train 28: YAML writer spells non-string dict keys with DictKeyText instead of throwing --- src/serialize_filters.cpp | 19 ++++++++++--------- test/filters_test.cpp | 1 + 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index d32df99a1..b948f76d1 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -8,6 +8,7 @@ #include "recursion_guard.h" #include "render_context.h" #include "string_builder.h" +#include "template_dict.h" #include "undefined.h" #include "unicode_printable.h" #include "value_visitors.h" @@ -909,15 +910,14 @@ class YamlWriter // namespace(), cycler() and joiner() are objects, not data throw std::runtime_error("Object of type "s + objectClass + " is not YAML serializable"); } - // Keys other than strings (docs/tasks/0036) are not written as YAML mapping keys + // A key that is not a string is written as its str() spelling (DictKeyText), as at the + // public boundary (docs/tasks/0036) + const auto entries = map.GetEntries(); std::vector keys; - for (const auto& entry : map.GetEntries()) + keys.reserve(entries.size()); + for (const auto& entry : entries) { - if (!IsStringValue(entry.key)) - { - throw std::runtime_error("Only string dictionary keys are YAML serializable"); - } - keys.push_back(AsString(entry.key)); + keys.push_back(IsStringValue(entry.key) ? AsString(entry.key) : DictKeyText(entry.key)); } if (keys.empty()) { @@ -926,8 +926,9 @@ class YamlWriter return; } bool isFirst = true; - for (const auto& key : keys) + for (size_t idx = 0; idx != keys.size(); ++idx) { + const auto& key = keys[idx]; if (isFirst) { StartNode(pos, indent, true); @@ -950,7 +951,7 @@ class YamlWriter WriteString(key); } m_out.push_back(':'); - WriteNode(map.GetValueByName(key), indent + 2, Position::MappingValue); + WriteNode(entries[idx].value, indent + 2, Position::MappingValue); } } diff --git a/test/filters_test.cpp b/test/filters_test.cpp index e5399913a..8244b33c6 100644 --- a/test/filters_test.cpp +++ b/test/filters_test.cpp @@ -690,6 +690,7 @@ INSTANTIATE_TEST_SUITE_P(EscapeCpp, FilterGenericTest, ::testing::Values( INSTANTIATE_TEST_SUITE_P(ToYaml, FilterGenericTest, ::testing::Values( InputOutputPair{"{'a': 1, 'b': [1, 2], 'c': {'d': none, 'e': 'x y'}} | toyaml", "a: 1\nb:\n - 1\n - 2\nc:\n d: null\n e: x y"}, + InputOutputPair{"{1: 'a', (1, 2): 'b', none: 'c'} | toyaml", "\"1\": a\n\"(1, 2)\": b\nNone: c"}, InputOutputPair{"[1, 'two', [3, 4], {'k': true}, [], {}] | toyaml", "- 1\n- two\n- - 3\n - 4\n- k: true\n- []\n- {}"}, InputOutputPair{"[{'a': 1, 'b': {'c': 2}}] | toyaml", "- a: 1\n b:\n c: 2"}, From b051f0404e482878c5cad58a208d421ccda8268f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 06:35:51 +0000 Subject: [PATCH 18/18] Train 28: nested pointer test and count() == 0 in the key walk (clang-tidy) --- src/value_methods.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/value_methods.cpp b/src/value_methods.cpp index ef3589153..179e7ec1b 100644 --- a/src/value_methods.cpp +++ b/src/value_methods.cpp @@ -1723,12 +1723,15 @@ bool ReachesThroughOpaque(const InternalValue& root, const void* target) { auto* items = map->GetMutableItems(); // A key that is not a string keeps its first spelling, which CycleCollector follows too - if (items && items->HasKeyValues() && !visited[viaOpaque ? 1 : 0].count(items)) + if (items) { - for (const auto& entry : *items) + if (items->HasKeyValues() && visited[viaOpaque ? 1 : 0].count(items) == 0) { - keys.push_back(items->KeyOf(entry.first)); - queueValue(keys.back(), viaOpaque); + for (const auto& entry : *items) + { + keys.push_back(items->KeyOf(entry.first)); + queueValue(keys.back(), viaOpaque); + } } } found = QueueItems(map->GetMutableItems(), target, viaOpaque, visited, queuePairValue);