From 1f3846437183fcc2f3df5a1698c2f16130586f1d Mon Sep 17 00:00:00 2001 From: "sanjul.sharma" Date: Tue, 20 Oct 2026 14:13:58 +0530 Subject: [PATCH] ADP-1966 | Escape regex metacharacters in Postgres LIKE filter values Postgres's `~*` operator (used to implement the LIKE/"contains" filter) is a case-insensitive, unanchored regex match using POSIX ARE, whereas MongoDB's $regex uses PCRE2. PCRE2 tolerates malformed patterns (e.g. an unmatched quantifier brace like "{{") by treating them literally, while Postgres's ARE engine hard-errors with "invalid regular expression: quantifier operand invalid" on the same input - breaking LIKE/"contains" filters on Postgres for values containing regex metacharacters (e.g. API Inventory domain search values like "{{" or "${${8gs"). Add a validity gate (PostgresUtils#sanitizeRegexValue) that passes a LIKE value through unchanged if it is already a valid regex (preserving existing regex-filter semantics, e.g. ".*amp.*", "^Sun"), and otherwise escapes POSIX ARE metacharacters so the value is matched literally instead of erroring out. Apply this in both LIKE code paths: the legacy Filter/Op parser and the v1 expression parser (PostgresLikeRelationalFilterParser), the latter being the actual path hit by the reported production stacktrace. Also drop the now-redundant ".*.*" wrapping previously applied to LIKE values before the ~* search, since `~*` is already an unanchored substring regex match. Note: for a value that is not a valid regex in either engine but IS syntactically valid Perl/PCRE2 (e.g. "${${8gs", which PCRE2 compiles but matches 0 documents because "$" is treated as an end-of-line anchor), this introduces a narrow Mongo/Postgres result divergence (Mongo: 0 matches, Postgres: 1 literal match) - a deliberate, documented trade-off versus leaving Postgres's LIKE operator broken for such inputs. Co-Authored-By: Claude Sonnet 5 AI-Session-Id: f9a3360e-ecb3-4388-beeb-682ed3b1d400 AI-Tool: claude-code AI-Model: unknown --- .../documentstore/DocStoreQueryV1Test.java | 48 +++++++ .../resources/query/collection_data.json | 3 +- .../PostgresLikeRelationalFilterParser.java | 10 +- .../postgres/utils/PostgresUtils.java | 53 +++++++- .../postgres/PostgresQueryParserTest.java | 33 +++++ ...ostgresLikeRelationalFilterParserTest.java | 121 ++++++++++++++++++ 6 files changed, 258 insertions(+), 10 deletions(-) create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParserTest.java diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 01e000537..bad72fd6d 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -1916,6 +1916,54 @@ public void testUnnestWithRegularFilterAndNullAndEmptyPreservedAtFirstLevel(Stri dataStoreName, iterator, "query/unwind_preserve_with_regular_filter_first_level.json", 3); } + @ParameterizedTest + @ArgumentsSource(AllProvider.class) + void testLikeFilterWithRegexMetacharactersInValue(String dataStoreName) { + // Reproduces a bug where a LIKE/"contains" filter whose value contains regex + // metacharacters (e.g. "{{") works on Mongo (lenient PCRE-style regex engine, where an + // unmatched "{" is treated as a literal) but fails on Postgres, whose `~*` operator uses + // POSIX ARE and throws "invalid regular expression: quantifier operand invalid" for the + // same value. Postgres now sanitizes values that are not valid regexes (see + // PostgresUtils#sanitizeRegexValue) so they are matched literally instead of erroring out. + Collection collection = getCollection(dataStoreName); + + Query bracesQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("notes"), LIKE, ConstantExpression.of("{{"))) + .build(); + + // Only id=8 has a "notes" field, containing the literal substring "{{". PCRE (Mongo) treats + // the unmatched "{{" as literal, and Postgres now escapes it to match literally too, so both + // data stores agree here. + long bracesCount = collection.count(bracesQuery); + assertEquals(1, bracesCount); + + Query dollarBraceQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("notes"), LIKE, ConstantExpression.of("${${8gs"))) + .build(); + + // "${${8gs" is NOT parity between the two data stores, and that is expected/correct, not a + // bug: it is a syntactically valid PCRE pattern (Mongo's engine), but the "$" appearing + // mid-pattern is an end-of-line anchor rather than a literal "$", so the pattern can never + // actually match anywhere in the "notes" value and Mongo legitimately returns 0 matches. + // Postgres/Java's regex engine rejects this same string outright as an illegal repetition + // (unmatched quantifier brace), so PostgresUtils#sanitizeRegexValue falls back to escaping it + // and matching it literally, which DOES match the "notes" value, returning 1. This is a + // genuine, pre-existing semantic difference between a lenient-but-unsatisfiable PCRE pattern + // and a literal-string fallback - not something the LIKE-filter fix can or should paper over. + long dollarBraceCount = collection.count(dollarBraceQuery); + if (MONGO_STORE.equals(dataStoreName)) { + assertEquals(0, dollarBraceCount); + } else { + assertEquals(1, dollarBraceCount); + } + } + @ParameterizedTest @ArgumentsSource(AllProvider.class) void testContainsAndUnnestFilters(String dataStoreName) throws IOException { diff --git a/document-store/src/integrationTest/resources/query/collection_data.json b/document-store/src/integrationTest/resources/query/collection_data.json index 3a389f38b..70e69c945 100644 --- a/document-store/src/integrationTest/resources/query/collection_data.json +++ b/document-store/src/integrationTest/resources/query/collection_data.json @@ -182,6 +182,7 @@ "item": "Soap", "price": 10, "quantity": 5, - "date": "2016-02-06T20:20:13Z" + "date": "2016-02-06T20:20:13Z", + "notes": "weird {{bracket}} and ${${8gs} value" } ] diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParser.java index a66070df9..9102221de 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParser.java @@ -1,6 +1,7 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; class PostgresLikeRelationalFilterParser implements PostgresRelationalFilterParser { @@ -10,9 +11,12 @@ public String parse( final Object parsedLhs = expression.getLhs().accept(context.lhsParser()); final Object parsedRhs = expression.getRhs().accept(context.rhsParser()); - // Append ".*" at beginning and end of value to do a regex - // Like operator is only applicable for string RHS. Hence, convert the RHS to string - final String rhsValue = ".*" + parsedRhs + ".*"; + // Like operator is only applicable for string RHS. Hence, convert the RHS to string. + // Postgres's `~*` is already an unanchored, case-insensitive regex search (not a full-string + // match), so no ".*" wrapping is needed. If the RHS is a valid regex, pass it through + // unchanged; otherwise escape POSIX ARE metacharacters so it is matched literally instead of + // making Postgres's `~*` operator error out (see PostgresUtils#sanitizeRegexValue). + final String rhsValue = PostgresUtils.sanitizeRegexValue(parsedRhs); context.getParamsBuilder().addObjectParam(rhsValue); // Case-insensitive regex search diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java index 5e8e64cf7..10f8712a8 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java @@ -23,6 +23,8 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import lombok.extern.slf4j.Slf4j; @@ -43,6 +45,9 @@ public class PostgresUtils { private static final String JSON_DATA_ACCESSOR = "->>"; private static final String DOT_STR = "_dot_"; private static final String DOT = "."; + // Every character that is special at the regex pattern level, for use by + // #escapeRegexMetacharacters(Object). + private static final String REGEX_METACHARACTERS = "\\.^$*+?()[]{}|"; public static final Set OUTER_COLUMNS = new TreeSet<>(List.of(ID, CREATED_AT, UPDATED_AT)); @@ -179,6 +184,42 @@ public static String prepareParameterizedStringForJsonList( return "(" + collect + ")"; } + /** + * Escapes POSIX ARE metacharacters so the value matches literally under Postgres's {@code ~*} + * operator. Fallback only - use {@link #sanitizeRegexValue(Object)} to decide whether escaping is + * needed, since a value that is itself a valid regex must be passed through unescaped. + */ + public static String escapeRegexMetacharacters(final Object value) { + // String.valueOf() preserves the pre-existing null-handling behaviour (literal "null"). + final String stringValue = String.valueOf(value); + final StringBuilder escaped = new StringBuilder(stringValue.length()); + for (int i = 0; i < stringValue.length(); i++) { + final char c = stringValue.charAt(i); + // Built char by char, so an inserted backslash is never re-escaped. + if (REGEX_METACHARACTERS.indexOf(c) >= 0) { + escaped.append('\\'); + } + escaped.append(c); + } + return escaped.toString(); + } + + /** + * {@code LIKE} is actually a regex match (see {@code RelationalOperator.LIKE}), so a value that + * is already a valid regex is passed through unchanged; otherwise it is escaped via {@link + * #escapeRegexMetacharacters(Object)} so Postgres's stricter {@code ~*} engine matches it + * literally instead of erroring out on malformed patterns that Mongo's PCRE tolerates. + */ + public static String sanitizeRegexValue(final Object value) { + final String stringValue = String.valueOf(value); + try { + Pattern.compile(stringValue); + return stringValue; + } catch (final PatternSyntaxException e) { + return escapeRegexMetacharacters(stringValue); + } + } + public static String parseNonCompositeFilterWithCasting( String fieldName, String columnName, String op, Object value, Builder paramsBuilder) { String parsedExpression = @@ -235,10 +276,10 @@ public static String parseNonCompositeFilter( break; case "LIKE": case "~": - // Case insensitive regex search, Append % at beginning and end of value to do a regex - // search + // Postgres's `~*` is an unanchored, case-insensitive regex search (not a full-string + // match), so no additional ".*" wrapping is needed to do a "contains" style search. sqlOperator = " ~* "; - value = ".*" + value + ".*"; + value = sanitizeRegexValue(value); break; case "NOT_IN": case "NOT IN": @@ -444,10 +485,10 @@ public static String prepareParsedNonCompositeFilter( break; case "LIKE": case "~": - // Case insensitive regex search, Append % at beginning and end of value to do a regex - // search + // Postgres's `~*` is an unanchored, case-insensitive regex search (not a full-string + // match), so no additional ".*" wrapping is needed to do a "contains" style search. sqlOperator = " ~* "; - value = ".*" + value + ".*"; + value = sanitizeRegexValue(value); break; case "NOT_IN": case "NOT IN": diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/PostgresQueryParserTest.java index c0e377657..fcd58c810 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/PostgresQueryParserTest.java @@ -101,6 +101,39 @@ void testParseNonCompositeFilter() { Assertions.assertEquals(ID + " ~* ?", query); } + { + // "a{{b" is not a valid regex (unmatched quantifier brace), so POSIX ARE metacharacters + // in the LIKE value must be escaped so that the value is matched literally instead of + // being rejected outright by Postgres's strict ~* operator. + Filter filter = new Filter(Filter.Op.LIKE, ID, "a{{b"); + Params.Builder paramsBuilder = initParams(); + String query = + PostgresUtils.parseNonCompositeFilterWithCasting( + filter.getFieldName(), + PostgresUtils.DOCUMENT_COLUMN, + filter.getOp().toString(), + filter.getValue(), + paramsBuilder); + Assertions.assertEquals(ID + " ~* ?", query); + Assertions.assertEquals(Map.of(1, "a\\{\\{b"), paramsBuilder.build().getObjectParams()); + } + + { + // A value that is already a valid regex (e.g. "^Sun") must be passed through unchanged, + // since LIKE is a case-insensitive regex match, not a literal "contains". + Filter filter = new Filter(Filter.Op.LIKE, ID, "^Sun"); + Params.Builder paramsBuilder = initParams(); + String query = + PostgresUtils.parseNonCompositeFilterWithCasting( + filter.getFieldName(), + PostgresUtils.DOCUMENT_COLUMN, + filter.getOp().toString(), + filter.getValue(), + paramsBuilder); + Assertions.assertEquals(ID + " ~* ?", query); + Assertions.assertEquals(Map.of(1, "^Sun"), paramsBuilder.build().getObjectParams()); + } + { Filter filter = new Filter(Filter.Op.IN, ID, List.of("abc", "xyz")); String query = diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParserTest.java new file mode 100644 index 000000000..c7bb96f19 --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresLikeRelationalFilterParserTest.java @@ -0,0 +1,121 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Map; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; +import org.hypertrace.core.documentstore.expression.operators.RelationalOperator; +import org.hypertrace.core.documentstore.postgres.Params; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser.PostgresRelationalFilterContext; +import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresSelectTypeExpressionVisitor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PostgresLikeRelationalFilterParserTest { + + private PostgresLikeRelationalFilterParser parser; + private PostgresRelationalFilterContext context; + private PostgresSelectTypeExpressionVisitor lhsParser; + private PostgresSelectTypeExpressionVisitor rhsParser; + private Params.Builder paramsBuilder; + + @BeforeEach + void setUp() { + parser = new PostgresLikeRelationalFilterParser(); + context = mock(PostgresRelationalFilterContext.class); + lhsParser = mock(PostgresSelectTypeExpressionVisitor.class); + rhsParser = mock(PostgresSelectTypeExpressionVisitor.class); + paramsBuilder = Params.newBuilder(); + + when(context.lhsParser()).thenReturn(lhsParser); + when(context.rhsParser()).thenReturn(rhsParser); + when(context.getParamsBuilder()).thenReturn(paramsBuilder); + } + + @Test + void testParse_plainValue() { + IdentifierExpression lhs = IdentifierExpression.of("key1"); + ConstantExpression rhs = ConstantExpression.of("abc"); + RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'"); + when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("abc"); + + String result = parser.parse(expression, context); + + assertEquals("document->>'key1' ~* ?", result); + assertEquals(Map.of(1, "abc"), paramsBuilder.build().getObjectParams()); + } + + @Test + void testParse_valueWithRegexMetacharacters() { + // POSIX ARE metacharacters (e.g. "{{") must be escaped so Postgres's strict ~* operator + // matches the value literally instead of throwing "invalid regular expression: quantifier + // operand invalid". + IdentifierExpression lhs = IdentifierExpression.of("key1"); + ConstantExpression rhs = ConstantExpression.of("a{{b"); + RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'"); + when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("a{{b"); + + String result = parser.parse(expression, context); + + assertEquals("document->>'key1' ~* ?", result); + assertEquals(Map.of(1, "a\\{\\{b"), paramsBuilder.build().getObjectParams()); + } + + @Test + void testParse_valueWithAllMetacharacters() { + // "${${8gs" is not a valid Java/Postgres regex (unmatched quantifier brace), so it must be + // escaped and matched literally, same as "a{{b" above. + IdentifierExpression lhs = IdentifierExpression.of("key1"); + ConstantExpression rhs = ConstantExpression.of("${${8gs"); + RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'"); + when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("${${8gs"); + + String result = parser.parse(expression, context); + + assertEquals("document->>'key1' ~* ?", result); + assertEquals(Map.of(1, "\\$\\{\\$\\{8gs"), paramsBuilder.build().getObjectParams()); + } + + @Test + void testParse_valueIsValidRegex_passedThroughUnchanged() { + // A value that is already a valid regex must be passed through unchanged rather than + // escaped, since LIKE is a case-insensitive regex match (not literal "contains"). + IdentifierExpression lhs = IdentifierExpression.of("key1"); + ConstantExpression rhs = ConstantExpression.of(".*amp.*"); + RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'"); + when(rhsParser.visit(any(ConstantExpression.class))).thenReturn(".*amp.*"); + + String result = parser.parse(expression, context); + + assertEquals("document->>'key1' ~* ?", result); + assertEquals(Map.of(1, ".*amp.*"), paramsBuilder.build().getObjectParams()); + } + + @Test + void testParse_valueIsValidAnchoredRegex_passedThroughUnchanged() { + IdentifierExpression lhs = IdentifierExpression.of("key1"); + ConstantExpression rhs = ConstantExpression.of("^Sun"); + RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs); + + when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'"); + when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("^Sun"); + + String result = parser.parse(expression, context); + + assertEquals("document->>'key1' ~* ?", result); + assertEquals(Map.of(1, "^Sun"), paramsBuilder.build().getObjectParams()); + } +}