diff --git a/packages/mcp-core/src/skillDefinitions.json b/packages/mcp-core/src/skillDefinitions.json index 2a8a8e62d..ad3b73447 100644 --- a/packages/mcp-core/src/skillDefinitions.json +++ b/packages/mcp-core/src/skillDefinitions.json @@ -214,7 +214,7 @@ }, { "name": "search_logs", - "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n", + "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n- Sentry search syntax for logs includes RE2 regex filters, never quoted: `message://^Timeout after \\d+ms//`.\n", "requiredScopes": ["event:read"] }, { @@ -309,7 +309,7 @@ }, { "name": "search_logs", - "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n", + "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n- Sentry search syntax for logs includes RE2 regex filters, never quoted: `message://^Timeout after \\d+ms//`.\n", "requiredScopes": ["event:read"] }, { @@ -475,7 +475,7 @@ }, { "name": "search_logs", - "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n", + "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n- Sentry search syntax for logs includes RE2 regex filters, never quoted: `message://^Timeout after \\d+ms//`.\n", "requiredScopes": ["event:read"] }, { diff --git a/packages/mcp-core/src/toolDefinitions.json b/packages/mcp-core/src/toolDefinitions.json index 76a29a173..e686c09ba 100644 --- a/packages/mcp-core/src/toolDefinitions.json +++ b/packages/mcp-core/src/toolDefinitions.json @@ -7832,7 +7832,7 @@ }, { "name": "search_logs", - "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n", + "description": "Search Sentry logs: application log entries, including error- and warning-severity log messages. Use for log counts, statistics, trends, and individual log lines.\n\n`query` is natural language (preferred) or Sentry search syntax; a configured agent translates it into query, fields, and sort.\n\nSupports aggregations ('warning logs by service'), individual entries ('error logs from the last hour'), and time series ('error logs per hour').\n\nNOT for exceptions/crashes (use search_errors). For requests or spans whose trace also has a matching log, use search_traces.\n\n\nsearch_logs(organizationSlug='my-org', query='error logs from the last hour')\nsearch_logs(organizationSlug='my-org', query='logs mentioning payment timeout in production')\nsearch_logs(organizationSlug='my-org', query='count warning logs by service over 7 days')\n\n\n\n- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.\n- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.\n- Sentry search syntax for logs includes RE2 regex filters, never quoted: `message://^Timeout after \\d+ms//`.\n", "inputSchema": { "type": "object", "properties": { diff --git a/packages/mcp-core/src/tools/catalog/search-events-environment-note.test.ts b/packages/mcp-core/src/tools/catalog/search-events-environment-note.test.ts index b8b4a05a1..612be897e 100644 --- a/packages/mcp-core/src/tools/catalog/search-events-environment-note.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-events-environment-note.test.ts @@ -59,6 +59,21 @@ describe("collectRequestedEnvironments", () => { ), ).toEqual(["qa"]); }); + + it("keeps regex values with spaces and apostrophes as one token", () => { + expect( + collectRequestedEnvironments( + null, + "message://can't connect// environment:qa", + ), + ).toEqual(["qa"]); + expect( + collectRequestedEnvironments(null, "message://a environment:foo b//"), + ).toEqual([]); + expect( + collectRequestedEnvironments(null, "(message://a environment:foo b//)"), + ).toEqual([]); + }); }); describe("formatUnknownEnvironmentNote", () => { diff --git a/packages/mcp-core/src/tools/catalog/search-events.test.ts b/packages/mcp-core/src/tools/catalog/search-events.test.ts index ab030f013..33766f851 100644 --- a/packages/mcp-core/src/tools/catalog/search-events.test.ts +++ b/packages/mcp-core/src/tools/catalog/search-events.test.ts @@ -3260,6 +3260,115 @@ describe("search_events", () => { expect(result).toContain("TimeoutError"); }); + describe("with regex log queries", () => { + const regexSearchParams = { + organizationSlug: "test-org", + regionUrl: null, + projectSlug: null, + dataset: "logs" as const, + fields: null, + sort: null, + period: "24h", + limit: 10, + includeExplanation: false, + }; + const regexSearchContext = { + constraints: { + organizationSlug: null, + regionUrl: null, + projectSlug: null, + }, + accessToken: "test-token", + userId: "1", + }; + + const captureEventsQueries = () => { + const queries: Array = []; + mswServer.use( + http.get( + "https://sentry.io/api/0/organizations/test-org/events/", + ({ request }) => { + queries.push(new URL(request.url).searchParams.get("query")); + return HttpResponse.json({ data: [] }); + }, + ), + ); + return queries; + }; + + it("runs a regex-only query as written without the agent when fields and sort are explicit", async () => { + const queries = captureEventsQueries(); + + await searchEvents.handler( + { + ...regexSearchParams, + query: "message://^Timeout after \\d+ms//", + fields: ["timestamp", "message"], + sort: "-timestamp", + }, + regexSearchContext, + ); + + expect(mockGenerateText).not.toHaveBeenCalled(); + expect(queries).toEqual(["message://^Timeout after \\d+ms//"]); + }); + + it("keeps the regex filter when the agent rewrites it into a wildcard", async () => { + mockGenerateText.mockResolvedValueOnce( + mockAIResponse("logs", 'message:"*Timeout after*"'), + ); + const queries = captureEventsQueries(); + + await searchEvents.handler( + { ...regexSearchParams, query: "message://^Timeout after \\d+ms//" }, + regexSearchContext, + ); + + expect(mockGenerateText).toHaveBeenCalledTimes(1); + expect(queries).toEqual(["message://^Timeout after \\d+ms//"]); + }); + + it("accepts agent repairs that keep a regex value with spaces and apostrophes", async () => { + mockGenerateText.mockResolvedValueOnce( + mockAIResponse( + "logs", + "severity:error message://can't connect to \\w+// has:trace", + ), + ); + const queries = captureEventsQueries(); + + await searchEvents.handler( + { + ...regexSearchParams, + query: "message://can't connect to \\w+// severity:error", + }, + regexSearchContext, + ); + + expect(queries).toEqual([ + "severity:error message://can't connect to \\w+// has:trace", + ]); + }); + + it("uses the agent's rewrite of regex syntax outside logs", async () => { + mockGenerateText.mockResolvedValueOnce( + mockAIResponse("spans", 'span.description:"GET /api/*"'), + ); + const queries = captureEventsQueries(); + + await searchEvents.handler( + { + ...regexSearchParams, + dataset: "spans", + query: "span.description://^GET \\/api\\/\\d+//", + }, + regexSearchContext, + ); + + expect(queries).toEqual(['span.description:"GET /api/*"']); + }); + }); + it("keeps caller fields when the agent returns an empty fields array", async () => { let eventsRequestUrl: URL | undefined; @@ -4163,6 +4272,10 @@ describe("search_events", () => { it.each([ ["a structured query", { query: "span.op:http.client" }], + [ + "a regex-only logs query", + { dataset: "logs" as const, query: "message://^Timeout//" }, + ], ["explicit fields", { fields: ["span.description", "count()"] }], ["an explicit sort", { sort: "-count()" }], ])("should skip Seer for %s", async (_, overrides) => { diff --git a/packages/mcp-core/src/tools/catalog/search-logs.ts b/packages/mcp-core/src/tools/catalog/search-logs.ts index 1a60689ed..bd98ca29d 100644 --- a/packages/mcp-core/src/tools/catalog/search-logs.ts +++ b/packages/mcp-core/src/tools/catalog/search-logs.ts @@ -27,6 +27,7 @@ export default defineTool({ "", "- name/otherName notation means /; parse it directly, don't call find_organizations/find_projects.", "- Natural language is usually enough. Only pass fields/sort when you need exact columns or ordering.", + "- Sentry search syntax for logs includes RE2 regex filters, never quoted: `message://^Timeout after \\d+ms//`.", "", ].join("\n"), inputSchema: buildDatasetSearchInputSchema(), diff --git a/packages/mcp-core/src/tools/support/search-events/search.ts b/packages/mcp-core/src/tools/support/search-events/search.ts index 508dd4513..49931d663 100644 --- a/packages/mcp-core/src/tools/support/search-events/search.ts +++ b/packages/mcp-core/src/tools/support/search-events/search.ts @@ -52,6 +52,7 @@ import { isAggregateQuery, isSemanticFilterDowngrade, looksLikeSentrySearchSyntax, + readRegexFilterValue, recordEventsSearchValidationTelemetry, validateEventsSearch, } from "./utils"; @@ -262,7 +263,8 @@ function tokenizeSearchQuery(query: string): string[] { let quote: '"' | "'" | null = null; let escaped = false; - for (const char of query) { + for (let i = 0; i < query.length; i += 1) { + const char = query[i]; if (escaped) { currentToken += char; escaped = false; @@ -283,6 +285,13 @@ function tokenizeSearchQuery(query: string): string[] { continue; } + const regexValue = readRegexFilterValue(query, i); + if (regexValue) { + currentToken += regexValue; + i += regexValue.length - 1; + continue; + } + if (char === '"' || char === "'") { currentToken += char; quote = char; @@ -322,6 +331,7 @@ function choosePreservingRepairedQuery(params: { originalQuery: string; repairedQuery?: string | null; filter?: string; + dataset: PublicEventsDataset | "replays"; }): string { const originalQuery = params.originalQuery.trim(); const repairedQuery = params.repairedQuery?.trim(); @@ -329,7 +339,7 @@ function choosePreservingRepairedQuery(params: { return appendSearchFilter(originalQuery, params.filter); } - if (isSemanticFilterDowngrade(originalQuery, repairedQuery)) { + if (isSemanticFilterDowngrade(originalQuery, repairedQuery, params.dataset)) { return appendSearchFilter(originalQuery, params.filter); } @@ -611,7 +621,10 @@ export async function runSearchEvents( setTargetTagsAndAttributes(params); const inputDataset = params.dataset ?? "errors"; - const hasStructuredQuery = looksLikeSentrySearchSyntax(params.query); + const hasStructuredQuery = looksLikeSentrySearchSyntax( + params.query, + inputDataset, + ); const canApplyEnvironmentFilter = inputDataset !== "replays" && isTraceItemDataset(inputDataset) && @@ -780,9 +793,14 @@ export async function runSearchEvents( originalQuery: params.query ?? "", repairedQuery: parsed.query, filter: environmentFilter, + dataset, }) - : looksLikeSentrySearchSyntax(params.query) && - isSemanticFilterDowngrade(params.query ?? "", parsed.query || "") + : looksLikeSentrySearchSyntax(params.query, dataset) && + isSemanticFilterDowngrade( + params.query ?? "", + parsed.query || "", + dataset, + ) ? (params.query ?? "") : parsed.query || ""; sortParam = diff --git a/packages/mcp-core/src/tools/support/search-events/utils.test.ts b/packages/mcp-core/src/tools/support/search-events/utils.test.ts index b156e46f9..23ee77616 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.test.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.test.ts @@ -292,14 +292,21 @@ describe("search query helpers", () => { it("should detect regex filters as structured Sentry search syntax", () => { expect( - looksLikeSentrySearchSyntax(String.raw`message://request took \d+ms//`), + looksLikeSentrySearchSyntax( + String.raw`message://request took \d+ms//`, + "logs", + ), ).toBe(true); expect( - looksLikeSentrySearchSyntax(String.raw`!message://worker \d+ ready//`), + looksLikeSentrySearchSyntax( + String.raw`!message://worker \d+ ready//`, + "logs", + ), ).toBe(true); expect( looksLikeSentrySearchSyntax( String.raw`message://https?://api\.example\.com//`, + "logs", ), ).toBe(true); }); @@ -314,6 +321,183 @@ describe("search query helpers", () => { expect(looksLikeSentrySearchSyntax("ERROR: service is down")).toBe(false); }); + it("should detect regex filters in logs queries", () => { + expect(looksLikeSentrySearchSyntax("message://^Timeout//", "logs")).toBe( + true, + ); + expect( + looksLikeSentrySearchSyntax("message://^Timeout after \\d+ms//", "logs"), + ).toBe(true); + expect( + looksLikeSentrySearchSyntax("!message://(?i)healthcheck//", "logs"), + ).toBe(true); + expect( + looksLikeSentrySearchSyntax("open http://example.com/a//b", "logs"), + ).toBe(false); + }); + + it("does not treat URLs as regex filters when they end in a regex terminator", () => { + const queries = [ + "errors from https://example.com/a// in checkout", + "requests to (http://example.com//)", + "uploads to s3://bucket and https://example.com//", + "errors from HTTP://example.com/a// in checkout", + ]; + + const results = queries.map((query) => + looksLikeSentrySearchSyntax(query, "logs"), + ); + + expect(results).toEqual([false, false, false, false]); + }); + + it("allows rewriting natural language log queries that contain URLs", () => { + const query = "errors from https://example.com/a// in checkout"; + + const downgrade = isSemanticFilterDowngrade( + query, + 'message:"*example.com/a*"', + "logs", + ); + + expect(downgrade).toBe(false); + }); + + it("should detect regex filters inside parentheses and on Sentry's other key shapes", () => { + expect(looksLikeSentrySearchSyntax("(message://a b//)", "logs")).toBe(true); + expect( + looksLikeSentrySearchSyntax("tags[App:Region]://^us-//", "logs"), + ).toBe(true); + expect(looksLikeSentrySearchSyntax("flags[Feature:X]://y//", "logs")).toBe( + true, + ); + expect( + looksLikeSentrySearchSyntax("tags[foo, string]://a b//", "logs"), + ).toBe(true); + expect(looksLikeSentrySearchSyntax('"mykey"://a b//', "logs")).toBe(true); + expect(looksLikeSentrySearchSyntax("arr[*]://a b//", "logs")).toBe(true); + expect(looksLikeSentrySearchSyntax('"Note"message://a b//', "logs")).toBe( + true, + ); + expect(looksLikeSentrySearchSyntax("(level:error)", "logs")).toBe(false); + }); + + it("should not treat regex filters as search syntax outside logs", () => { + expect(looksLikeSentrySearchSyntax("message://^Timeout//")).toBe(false); + expect( + looksLikeSentrySearchSyntax("span.description://^GET \\/api//", "spans"), + ).toBe(false); + expect(looksLikeSentrySearchSyntax("message://^Timeout//", "errors")).toBe( + false, + ); + }); + + it("detects regex filters rewritten into plain or wildcard filters", () => { + expect( + isSemanticFilterDowngrade( + "message://^Timeout//", + 'message:"*Timeout*"', + "logs", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "message://can't connect to \\w+//", + "message:*connect*", + "logs", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "custom://^order-\\d+$// level:error", + 'level:error message:"*order-*"', + "logs", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "custom://^order-\\d+$//", + "custom:order-*", + "logs", + ), + ).toBe(true); + + expect( + isSemanticFilterDowngrade( + "message://^Timeout//", + "message://^Timeout// severity:error", + "logs", + ), + ).toBe(false); + expect( + isSemanticFilterDowngrade( + "custom://^order-\\d+$//", + "tags[custom]://^order-\\d+$//", + "logs", + ), + ).toBe(false); + }); + + it("detects regex filter downgrades inside parentheses and on colon keys", () => { + expect( + isSemanticFilterDowngrade( + "severity:error (message://^Timeout after \\d+ms//)", + 'severity:error message:"*Timeout after*"', + "logs", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "tags[sentry:user]://^id \\d+//", + "tags[sentry:user]://^id//", + "logs", + ), + ).toBe(true); + }); + + it("detects regex filters rewritten into wildcards on array, quoted, and spaced keys", () => { + expect( + isSemanticFilterDowngrade("arr[*]://^a b//", "arr[*]:*a*", "logs"), + ).toBe(true); + expect( + isSemanticFilterDowngrade('"mykey"://^a b//', '"mykey":*a*', "logs"), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "tags[x, string]://^a b//", + 'tags[x, string]:"*a*"', + "logs", + ), + ).toBe(true); + }); + + it("detects regex filters whose pattern changed case", () => { + expect( + isSemanticFilterDowngrade( + "message://^Timeout//", + "message://^timeout//", + "logs", + ), + ).toBe(true); + }); + + it("allows rewriting regex filters outside logs", () => { + expect( + isSemanticFilterDowngrade( + "span.description://^GET \\/api\\/\\d+//", + 'span.description:"GET /api/*"', + "spans", + ), + ).toBe(false); + expect( + isSemanticFilterDowngrade( + "message://^Timeout//", + 'message:"*Timeout*"', + "errors", + ), + ).toBe(false); + }); + it("detects message full-text downgrades but allows real field renames", () => { expect( isSemanticFilterDowngrade("conv_id:ZYGC-86ZR", 'message:"*ZYGC-86ZR*"'), @@ -444,6 +628,40 @@ describe("search query helpers", () => { ), ).toBe(true); }); + + it("detects regex downgrades of structured filters on datasets other than logs", () => { + expect( + isSemanticFilterDowngrade( + "http.status_code:500", + "message://status 500//", + "errors", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "span.op:db", + "span.description://db query//", + "spans", + ), + ).toBe(false); + }); + + it("detects message regex downgrades when the value casing differs", () => { + expect( + isSemanticFilterDowngrade( + "transaction:CheckoutFlow", + "message://(?i)CheckoutFlow//", + "logs", + ), + ).toBe(true); + expect( + isSemanticFilterDowngrade( + "user.email:Foo@example.com", + "message://Foo@example.com// severity:error", + "logs", + ), + ).toBe(true); + }); }); describe("fetchCustomAttributes", () => { diff --git a/packages/mcp-core/src/tools/support/search-events/utils.ts b/packages/mcp-core/src/tools/support/search-events/utils.ts index 3e6b67daf..677c406c1 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.ts @@ -34,8 +34,25 @@ const DEFAULT_MAX_ARRAY_ITEMS = 20; // separates a key://pattern// filter from prose like http://example.com. const REGEX_FILTER_VALUE_SOURCE = String.raw`\/\/(?!\/\/(?:[\t\n )]|$))[^\n]{1,1024}?\/\/(?=[\t\n )]|$)`; const REGEX_FILTER_VALUE_PATTERN = new RegExp(`^${REGEX_FILTER_VALUE_SOURCE}`); +const SEARCH_FILTER_KEY_SOURCE = String.raw`(^|\s)!?(?[A-Za-z_][A-Za-z0-9_.[\],-]*):`; +// Sentry would parse `https://host/path// ` as a regex filter too, but no +// attribute is named after a URL scheme, so treat those as plain text. +const URL_SCHEME_KEY_SOURCE = + String.raw`(?:https?|wss?|s?ftp|file|s3|gs):`.replace( + /[a-z]/g, + (letter) => `[${letter}${letter.toUpperCase()}]`, + ); +const REGEX_FILTER_KEY_SOURCE = String.raw`(^|[\s()"])!?(?(?:tags|flags)\[[\w.:-]+(?: *, *(?:string|number|boolean|array))?\](?:\[\*\])?|"[\w.:-]+"(?:\[\*\])?|(?!${URL_SCHEME_KEY_SOURCE})[A-Za-z_][A-Za-z0-9_.[\],-]*(?:\[\*\])?):`; +const REGEX_FILTER_KEY_BEFORE_PATTERN = new RegExp( + `${REGEX_FILTER_KEY_SOURCE}$`, +); +const REGEX_FILTER_KEY_SCAN_LIMIT = 256; const SENTRY_SEARCH_TOKEN_PATTERN = new RegExp( - String.raw`(^|\s)!?([A-Za-z_][A-Za-z0-9_.[\],-]*):(?=\S)(?:(?!\/\/)|(?=${REGEX_FILTER_VALUE_SOURCE}))`, + String.raw`${SEARCH_FILTER_KEY_SOURCE}(?=\S)(?!\/\/)`, + "g", +); +const SENTRY_SEARCH_TOKEN_WITH_REGEX_PATTERN = new RegExp( + String.raw`${REGEX_FILTER_KEY_SOURCE}(?=${REGEX_FILTER_VALUE_SOURCE})|${SEARCH_FILTER_KEY_SOURCE}(?=\S)(?!\/\/)`, "g", ); const KNOWN_SENTRY_SEARCH_KEYS = new Set([ @@ -93,14 +110,31 @@ export function isAggregateQuery(fields: string[]): boolean { return fields.some((field) => field.includes("(") && field.includes(")")); } -export function looksLikeSentrySearchSyntax(query?: string): boolean { +type SearchSyntaxDataset = EventsDataset | "replays"; + +// Sentry only honors key://pattern// as a regex on logs; other datasets match +// the literal text //pattern//. +function searchTokenPattern(dataset?: SearchSyntaxDataset): RegExp { + return dataset === "logs" + ? SENTRY_SEARCH_TOKEN_WITH_REGEX_PATTERN + : SENTRY_SEARCH_TOKEN_PATTERN; +} + +export function looksLikeSentrySearchSyntax( + query?: string, + dataset?: SearchSyntaxDataset, +): boolean { const trimmedQuery = query?.trim(); if (!trimmedQuery) { return false; } - for (const match of trimmedQuery.matchAll(SENTRY_SEARCH_TOKEN_PATTERN)) { - const key = match[2]; + for (const match of trimmedQuery.matchAll(searchTokenPattern(dataset))) { + if (match.groups?.regexKey) { + return true; + } + + const key = match.groups?.key; if (!key) { continue; } @@ -125,13 +159,30 @@ export function looksLikeSentrySearchSyntax(query?: string): boolean { return false; } +export function readRegexFilterValue( + query: string, + index: number, +): string | undefined { + if (!query.startsWith("//", index) || query[index - 1] !== ":") { + return undefined; + } + // Callers probe every index, so only scan a bounded key-sized prefix; the + // NUL stands in for truncated text so `^` can't match mid-query. + const start = Math.max(0, index - REGEX_FILTER_KEY_SCAN_LIMIT); + const prefix = `${start > 0 ? "\0" : ""}${query.slice(start, index)}`; + if (!REGEX_FILTER_KEY_BEFORE_PATTERN.test(prefix)) { + return undefined; + } + return REGEX_FILTER_VALUE_PATTERN.exec(query.slice(index))?.[0]; +} + const FULL_TEXT_SEARCH_KEYS = new Set(["message", "log.body"]); /** - * Replace quoted regions and regex filter values with same-length placeholders - * so colons inside them are not treated as filter-key separators - * (e.g. transaction:"handle message:hello" or message://key:\d+//). - * Length is preserved so offsets still map back to the original query. + * Replace quoted regions and regex values with same-length placeholders so + * colons inside them are not treated as filter-key separators (e.g. + * transaction:"handle message:hello"). Length is preserved so offsets still + * map back to the original query. */ function maskQuotedRegions(query: string): string { let out = ""; @@ -139,16 +190,7 @@ function maskQuotedRegions(query: string): string { let escaped = false; for (let i = 0; i < query.length; i += 1) { - const char = query[i] ?? ""; - if (!quote && !escaped && char === "/" && query[i - 1] === ":") { - const regexValue = query.slice(i).match(REGEX_FILTER_VALUE_PATTERN)?.[0]; - if (regexValue) { - // Keep the // delimiters so the token pattern still sees a regex value. - out += `//${regexValue.slice(2, -2).replace(/[\s:"']/g, "x")}//`; - i += regexValue.length - 1; - continue; - } - } + const char = query[i]; if (escaped) { escaped = false; out += quote ? "x" : char; @@ -169,6 +211,12 @@ function maskQuotedRegions(query: string): string { } continue; } + const regexValue = readRegexFilterValue(query, i); + if (regexValue) { + out += `//${regexValue.slice(2, -2).replace(/[:\s]/g, "x")}//`; + i += regexValue.length - 1; + continue; + } if (char === '"' || char === "'") { quote = char; out += char; @@ -183,13 +231,10 @@ function maskQuotedRegions(query: string): string { type SearchFilterOccurrence = { key: string; value: string; + regex: boolean; }; function normalizeFilterValue(rawValue: string): string { - if (REGEX_FILTER_VALUE_PATTERN.test(rawValue)) { - return rawValue.slice(2, -2).trim().toLowerCase(); - } - return rawValue .replace(/^['"]|['"]$/g, "") .replace(/^\*+|\*+$/g, "") @@ -199,8 +244,7 @@ function normalizeFilterValue(rawValue: string): string { /** * Read one filter value starting at `valueStart` in the original query. - * Quoted and regex values keep interior whitespace; other values stop at - * whitespace. + * Quoted values keep interior whitespace; unquoted values stop at whitespace. */ function readRawFilterValue( query: string, @@ -210,13 +254,6 @@ function readRawFilterValue( return undefined; } - const regexValue = query - .slice(valueStart) - .match(REGEX_FILTER_VALUE_PATTERN)?.[0]; - if (regexValue) { - return regexValue; - } - const first = query[valueStart]; if (first === '"' || first === "'") { let escaped = false; @@ -251,15 +288,27 @@ function searchFilterOccurrences(query: string): SearchFilterOccurrence[] { const occurrences: SearchFilterOccurrence[] = []; const masked = maskQuotedRegions(query); - for (const match of masked.matchAll(SENTRY_SEARCH_TOKEN_PATTERN)) { - const key = match[2]?.toLowerCase(); + // Read key://pattern// on every dataset: outside logs it is a literal match, + // so a structured filter rewritten into one is still a downgrade. + for (const match of masked.matchAll(SENTRY_SEARCH_TOKEN_WITH_REGEX_PATTERN)) { + const key = (match.groups?.regexKey ?? match.groups?.key)?.toLowerCase(); if (!key || match.index === undefined) { continue; } // Read the real value from the original query at the same offset so quotes // and multi-word quoted values are preserved before normalization. - const valueStart = match.index + match[0].indexOf(":") + 1; + const valueStart = match.index + match[0].length; + const regexValue = readRegexFilterValue(query, valueStart); + if (regexValue) { + occurrences.push({ + key, + value: regexValue.slice(2, -2), + regex: true, + }); + continue; + } + const rawValue = readRawFilterValue(query, valueStart); if (!rawValue) { continue; @@ -270,7 +319,7 @@ function searchFilterOccurrences(query: string): SearchFilterOccurrence[] { continue; } - occurrences.push({ key, value }); + occurrences.push({ key, value, regex: false }); } return occurrences; @@ -289,7 +338,7 @@ function fullTextFilterValues(query: string): string[] { } function filterOccurrenceIdentity(occurrence: SearchFilterOccurrence): string { - return `${occurrence.key}\0${occurrence.value}`; + return `${occurrence.key}\0${occurrence.regex}\0${occurrence.value}`; } function escapeRegExp(value: string): string { @@ -317,7 +366,12 @@ function containsAsWholeToken(haystack: string, needle: string): boolean { } function isRelatedFilterValue(left: string, right: string): boolean { - return containsAsWholeToken(left, right) || containsAsWholeToken(right, left); + const lowerLeft = left.toLowerCase(); + const lowerRight = right.toLowerCase(); + return ( + containsAsWholeToken(lowerLeft, lowerRight) || + containsAsWholeToken(lowerRight, lowerLeft) + ); } /** @@ -348,9 +402,38 @@ function unmatchedStructuredFilters( return unmatched; } +function isRegexFilterDowngrade( + originalQuery: string, + repairedQuery: string, +): boolean { + const originalRegexFilters = searchFilterOccurrences(originalQuery).filter( + (occurrence) => occurrence.regex, + ); + if (originalRegexFilters.length === 0) { + return false; + } + + const repairedFilters = searchFilterOccurrences(repairedQuery); + const maskedRepairedQuery = maskQuotedRegions(repairedQuery); + return unmatchedStructuredFilters(originalRegexFilters, repairedFilters).some( + (filter) => + new RegExp( + String.raw`(?:^|[\s()"])!?${escapeRegExp(filter.key)}:(?!\/\/)`, + "i", + ).test(maskedRepairedQuery) || + repairedFilters.some( + (repaired) => + repaired.key === filter.key || + (FULL_TEXT_SEARCH_KEYS.has(repaired.key) && + isRelatedFilterValue(filter.value, repaired.value)), + ), + ); +} + /** * True when a structured field:value filter was replaced with message/log.body - * full-text matching (false-success path). Allows real attribute renames. + * full-text matching (false-success path), or a logs `key://pattern//` regex + * filter was replaced with a plain or wildcard filter. Allows real attribute renames. * * Uses multiset key+value matching so dropping one of several identical keys * (e.g. `custom:foo custom:bar` → `custom:bar message:"*foo*"`) is still caught. @@ -360,11 +443,19 @@ function unmatchedStructuredFilters( export function isSemanticFilterDowngrade( originalQuery: string, repairedQuery: string, + dataset?: SearchSyntaxDataset, ): boolean { - if (!looksLikeSentrySearchSyntax(originalQuery)) { + if (!looksLikeSentrySearchSyntax(originalQuery, dataset)) { return false; } + if ( + dataset === "logs" && + isRegexFilterDowngrade(originalQuery, repairedQuery) + ) { + return true; + } + const originalFilters = structuredFilterOccurrences(originalQuery); if (originalFilters.length === 0) { return false;