Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1916,6 +1916,54 @@ public void testUnnestWithRegularFilterAndNullAndEmptyPreservedAtFirstLevel(Stri
dataStoreName, iterator, "query/unwind_preserve_with_regular_filter_first_level.json", 3);
}

@ParameterizedTest
@ArgumentsSource(AllProvider.class)
void testLikeFilterWithRegexMetacharactersInValue(String dataStoreName) {
// Reproduces a bug where a LIKE/"contains" filter whose value contains regex
// metacharacters (e.g. "{{") works on Mongo (lenient PCRE-style regex engine, where an
// unmatched "{" is treated as a literal) but fails on Postgres, whose `~*` operator uses
// POSIX ARE and throws "invalid regular expression: quantifier operand invalid" for the
// same value. Postgres now sanitizes values that are not valid regexes (see
// PostgresUtils#sanitizeRegexValue) so they are matched literally instead of erroring out.
Collection collection = getCollection(dataStoreName);

Query bracesQuery =
Query.builder()
.setFilter(
RelationalExpression.of(
IdentifierExpression.of("notes"), LIKE, ConstantExpression.of("{{")))
.build();

// Only id=8 has a "notes" field, containing the literal substring "{{". PCRE (Mongo) treats
// the unmatched "{{" as literal, and Postgres now escapes it to match literally too, so both
// data stores agree here.
long bracesCount = collection.count(bracesQuery);
assertEquals(1, bracesCount);

Query dollarBraceQuery =
Query.builder()
.setFilter(
RelationalExpression.of(
IdentifierExpression.of("notes"), LIKE, ConstantExpression.of("${${8gs")))
.build();

// "${${8gs" is NOT parity between the two data stores, and that is expected/correct, not a
// bug: it is a syntactically valid PCRE pattern (Mongo's engine), but the "$" appearing
// mid-pattern is an end-of-line anchor rather than a literal "$", so the pattern can never
// actually match anywhere in the "notes" value and Mongo legitimately returns 0 matches.
// Postgres/Java's regex engine rejects this same string outright as an illegal repetition
// (unmatched quantifier brace), so PostgresUtils#sanitizeRegexValue falls back to escaping it
// and matching it literally, which DOES match the "notes" value, returning 1. This is a
// genuine, pre-existing semantic difference between a lenient-but-unsatisfiable PCRE pattern
// and a literal-string fallback - not something the LIKE-filter fix can or should paper over.
long dollarBraceCount = collection.count(dollarBraceQuery);
if (MONGO_STORE.equals(dataStoreName)) {
assertEquals(0, dollarBraceCount);
} else {
assertEquals(1, dollarBraceCount);
}
}

@ParameterizedTest
@ArgumentsSource(AllProvider.class)
void testContainsAndUnnestFilters(String dataStoreName) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@
"item": "Soap",
"price": 10,
"quantity": 5,
"date": "2016-02-06T20:20:13Z"
"date": "2016-02-06T20:20:13Z",
"notes": "weird {{bracket}} and ${${8gs} value"
}
]
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter;

import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils;

class PostgresLikeRelationalFilterParser implements PostgresRelationalFilterParser {

Expand All @@ -10,9 +11,12 @@ public String parse(
final Object parsedLhs = expression.getLhs().accept(context.lhsParser());
final Object parsedRhs = expression.getRhs().accept(context.rhsParser());

// Append ".*" at beginning and end of value to do a regex
// Like operator is only applicable for string RHS. Hence, convert the RHS to string
final String rhsValue = ".*" + parsedRhs + ".*";
// Like operator is only applicable for string RHS. Hence, convert the RHS to string.
// Postgres's `~*` is already an unanchored, case-insensitive regex search (not a full-string
// match), so no ".*" wrapping is needed. If the RHS is a valid regex, pass it through
// unchanged; otherwise escape POSIX ARE metacharacters so it is matched literally instead of
// making Postgres's `~*` operator error out (see PostgresUtils#sanitizeRegexValue).
final String rhsValue = PostgresUtils.sanitizeRegexValue(parsedRhs);
context.getParamsBuilder().addObjectParam(rhsValue);

// Case-insensitive regex search
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -43,6 +45,9 @@ public class PostgresUtils {
private static final String JSON_DATA_ACCESSOR = "->>";
private static final String DOT_STR = "_dot_";
private static final String DOT = ".";
// Every character that is special at the regex pattern level, for use by
// #escapeRegexMetacharacters(Object).
private static final String REGEX_METACHARACTERS = "\\.^$*+?()[]{}|";

public static final Set<String> OUTER_COLUMNS =
new TreeSet<>(List.of(ID, CREATED_AT, UPDATED_AT));
Expand Down Expand Up @@ -179,6 +184,42 @@ public static String prepareParameterizedStringForJsonList(
return "(" + collect + ")";
}

/**
* Escapes POSIX ARE metacharacters so the value matches literally under Postgres's {@code ~*}
* operator. Fallback only - use {@link #sanitizeRegexValue(Object)} to decide whether escaping is
* needed, since a value that is itself a valid regex must be passed through unescaped.
*/
public static String escapeRegexMetacharacters(final Object value) {
// String.valueOf() preserves the pre-existing null-handling behaviour (literal "null").
final String stringValue = String.valueOf(value);
final StringBuilder escaped = new StringBuilder(stringValue.length());
for (int i = 0; i < stringValue.length(); i++) {
final char c = stringValue.charAt(i);
// Built char by char, so an inserted backslash is never re-escaped.
if (REGEX_METACHARACTERS.indexOf(c) >= 0) {
escaped.append('\\');
}
escaped.append(c);
}
return escaped.toString();
}

/**
* {@code LIKE} is actually a regex match (see {@code RelationalOperator.LIKE}), so a value that
* is already a valid regex is passed through unchanged; otherwise it is escaped via {@link
* #escapeRegexMetacharacters(Object)} so Postgres's stricter {@code ~*} engine matches it
* literally instead of erroring out on malformed patterns that Mongo's PCRE tolerates.
*/
public static String sanitizeRegexValue(final Object value) {
final String stringValue = String.valueOf(value);
try {
Pattern.compile(stringValue);
return stringValue;
} catch (final PatternSyntaxException e) {
return escapeRegexMetacharacters(stringValue);
}
}

public static String parseNonCompositeFilterWithCasting(
String fieldName, String columnName, String op, Object value, Builder paramsBuilder) {
String parsedExpression =
Expand Down Expand Up @@ -235,10 +276,10 @@ public static String parseNonCompositeFilter(
break;
case "LIKE":
case "~":
// Case insensitive regex search, Append % at beginning and end of value to do a regex
// search
// Postgres's `~*` is an unanchored, case-insensitive regex search (not a full-string
// match), so no additional ".*" wrapping is needed to do a "contains" style search.
sqlOperator = " ~* ";
value = ".*" + value + ".*";
value = sanitizeRegexValue(value);
break;
case "NOT_IN":
case "NOT IN":
Expand Down Expand Up @@ -444,10 +485,10 @@ public static String prepareParsedNonCompositeFilter(
break;
case "LIKE":
case "~":
// Case insensitive regex search, Append % at beginning and end of value to do a regex
// search
// Postgres's `~*` is an unanchored, case-insensitive regex search (not a full-string
// match), so no additional ".*" wrapping is needed to do a "contains" style search.
sqlOperator = " ~* ";
value = ".*" + value + ".*";
value = sanitizeRegexValue(value);
break;
case "NOT_IN":
case "NOT IN":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,39 @@ void testParseNonCompositeFilter() {
Assertions.assertEquals(ID + " ~* ?", query);
}

{
// "a{{b" is not a valid regex (unmatched quantifier brace), so POSIX ARE metacharacters
// in the LIKE value must be escaped so that the value is matched literally instead of
// being rejected outright by Postgres's strict ~* operator.
Filter filter = new Filter(Filter.Op.LIKE, ID, "a{{b");
Params.Builder paramsBuilder = initParams();
String query =
PostgresUtils.parseNonCompositeFilterWithCasting(
filter.getFieldName(),
PostgresUtils.DOCUMENT_COLUMN,
filter.getOp().toString(),
filter.getValue(),
paramsBuilder);
Assertions.assertEquals(ID + " ~* ?", query);
Assertions.assertEquals(Map.of(1, "a\\{\\{b"), paramsBuilder.build().getObjectParams());
}

{
// A value that is already a valid regex (e.g. "^Sun") must be passed through unchanged,
// since LIKE is a case-insensitive regex match, not a literal "contains".
Filter filter = new Filter(Filter.Op.LIKE, ID, "^Sun");
Params.Builder paramsBuilder = initParams();
String query =
PostgresUtils.parseNonCompositeFilterWithCasting(
filter.getFieldName(),
PostgresUtils.DOCUMENT_COLUMN,
filter.getOp().toString(),
filter.getValue(),
paramsBuilder);
Assertions.assertEquals(ID + " ~* ?", query);
Assertions.assertEquals(Map.of(1, "^Sun"), paramsBuilder.build().getObjectParams());
}

{
Filter filter = new Filter(Filter.Op.IN, ID, List.of("abc", "xyz"));
String query =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import java.util.Map;
import org.hypertrace.core.documentstore.expression.impl.ConstantExpression;
import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression;
import org.hypertrace.core.documentstore.expression.impl.RelationalExpression;
import org.hypertrace.core.documentstore.expression.operators.RelationalOperator;
import org.hypertrace.core.documentstore.postgres.Params;
import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser.PostgresRelationalFilterContext;
import org.hypertrace.core.documentstore.postgres.query.v1.vistors.PostgresSelectTypeExpressionVisitor;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

class PostgresLikeRelationalFilterParserTest {

private PostgresLikeRelationalFilterParser parser;
private PostgresRelationalFilterContext context;
private PostgresSelectTypeExpressionVisitor lhsParser;
private PostgresSelectTypeExpressionVisitor rhsParser;
private Params.Builder paramsBuilder;

@BeforeEach
void setUp() {
parser = new PostgresLikeRelationalFilterParser();
context = mock(PostgresRelationalFilterContext.class);
lhsParser = mock(PostgresSelectTypeExpressionVisitor.class);
rhsParser = mock(PostgresSelectTypeExpressionVisitor.class);
paramsBuilder = Params.newBuilder();

when(context.lhsParser()).thenReturn(lhsParser);
when(context.rhsParser()).thenReturn(rhsParser);
when(context.getParamsBuilder()).thenReturn(paramsBuilder);
}

@Test
void testParse_plainValue() {
IdentifierExpression lhs = IdentifierExpression.of("key1");
ConstantExpression rhs = ConstantExpression.of("abc");
RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs);

when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'");
when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("abc");

String result = parser.parse(expression, context);

assertEquals("document->>'key1' ~* ?", result);
assertEquals(Map.of(1, "abc"), paramsBuilder.build().getObjectParams());
}

@Test
void testParse_valueWithRegexMetacharacters() {
// POSIX ARE metacharacters (e.g. "{{") must be escaped so Postgres's strict ~* operator
// matches the value literally instead of throwing "invalid regular expression: quantifier
// operand invalid".
IdentifierExpression lhs = IdentifierExpression.of("key1");
ConstantExpression rhs = ConstantExpression.of("a{{b");
RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs);

when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'");
when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("a{{b");

String result = parser.parse(expression, context);

assertEquals("document->>'key1' ~* ?", result);
assertEquals(Map.of(1, "a\\{\\{b"), paramsBuilder.build().getObjectParams());
}

@Test
void testParse_valueWithAllMetacharacters() {
// "${${8gs" is not a valid Java/Postgres regex (unmatched quantifier brace), so it must be
// escaped and matched literally, same as "a{{b" above.
IdentifierExpression lhs = IdentifierExpression.of("key1");
ConstantExpression rhs = ConstantExpression.of("${${8gs");
RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs);

when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'");
when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("${${8gs");

String result = parser.parse(expression, context);

assertEquals("document->>'key1' ~* ?", result);
assertEquals(Map.of(1, "\\$\\{\\$\\{8gs"), paramsBuilder.build().getObjectParams());
}

@Test
void testParse_valueIsValidRegex_passedThroughUnchanged() {
// A value that is already a valid regex must be passed through unchanged rather than
// escaped, since LIKE is a case-insensitive regex match (not literal "contains").
IdentifierExpression lhs = IdentifierExpression.of("key1");
ConstantExpression rhs = ConstantExpression.of(".*amp.*");
RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs);

when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'");
when(rhsParser.visit(any(ConstantExpression.class))).thenReturn(".*amp.*");

String result = parser.parse(expression, context);

assertEquals("document->>'key1' ~* ?", result);
assertEquals(Map.of(1, ".*amp.*"), paramsBuilder.build().getObjectParams());
}

@Test
void testParse_valueIsValidAnchoredRegex_passedThroughUnchanged() {
IdentifierExpression lhs = IdentifierExpression.of("key1");
ConstantExpression rhs = ConstantExpression.of("^Sun");
RelationalExpression expression = RelationalExpression.of(lhs, RelationalOperator.LIKE, rhs);

when(lhsParser.visit(any(IdentifierExpression.class))).thenReturn("document->>'key1'");
when(rhsParser.visit(any(ConstantExpression.class))).thenReturn("^Sun");

String result = parser.parse(expression, context);

assertEquals("document->>'key1' ~* ?", result);
assertEquals(Map.of(1, "^Sun"), paramsBuilder.build().getObjectParams());
}
}
Loading