diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 4500ac9..9ae512e 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -1791,10 +1791,19 @@ private function getColumnType(string $id, string $type = 'event'): string 'clientEngine', 'clientEngineVersion', 'deviceName', 'deviceBrand', 'deviceModel', 'hostname', 'ip', - // premium geo (lower-cardinality only; city/isp/AS org/connection org - // are high-cardinality and intentionally fall through to Nullable(String)) + // request attributes: only protocol is bounded. accept/acceptLanguage/ + // queryKeys hold raw, un-normalized caller input (unbounded distinct + // values), so they stay plain Nullable(String) + bloom_filter. + 'protocol', + // ip reputation verdict (placeholder): bounded enum + // (clean/low/suspicious/block) + 'ipReputation', + // premium geo (lower-cardinality only; city/isp/AS org/connection org and + // postalCode/latitude/longitude are high-cardinality and intentionally fall + // through to Nullable(String)) 'continentCode', 'subdivisions', 'connectionType', 'connectionUsageType', 'autonomousSystemNumber', + 'timeZone', 'weatherCode', // sdk identity 'sdk', 'sdkVersion', // gauge replica ordinal @@ -1843,6 +1852,8 @@ private function getColumnCodec(string $id): string 'autonomousSystemNumber', 'autonomousSystemOrganization', 'connectionType', 'connectionUsageType', 'connectionOrganization', 'sdk', 'sdkVersion', + // high-entropy, un-normalized request text + 'accept', 'acceptLanguage', 'queryKeys', ]; if (in_array($id, $zstdColumns, true)) { diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index 334feff..d993fe8 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -56,6 +56,9 @@ class Metric extends ArrayObject 'country', 'region', 'hostname', 'ip', // request attributes (firewall rule matching) 'protocol', 'accept', 'acceptLanguage', 'queryKeys', + // ip reputation (placeholder — populated later; bounded verdict enum: + // clean/low/suspicious/block) + 'ipReputation', // premium geo 'city', 'continentCode', 'subdivisions', 'postalCode', 'latitude', 'longitude', 'timeZone', 'weatherCode', @@ -92,6 +95,7 @@ class Metric extends ArrayObject * - teamId / teamInternalId: owning team identity * - country / region / hostname / ip: geographic + caller origin * - protocol / accept / acceptLanguage / queryKeys: request attributes (firewall rule matching) + * - ipReputation: IP reputation verdict (placeholder; clean/low/suspicious/block) * - city / continentCode / subdivisions: premium geo location fields * - postalCode / latitude / longitude / timeZone / weatherCode: premium geo location fields * - isp / autonomousSystemNumber / autonomousSystemOrganization: premium network origin @@ -658,6 +662,8 @@ public static function getEventSchema(): array $stringColumn('accept', 1024), $stringColumn('acceptLanguage', 256), $stringColumn('queryKeys', 1024), + // ip reputation (placeholder — bounded verdict enum) + $stringColumn('ipReputation', 32), // premium geo $stringColumn('city', 256), $stringColumn('continentCode', 2), @@ -776,20 +782,42 @@ public static function getEventIndexes(): array 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', 'osName', 'clientType', 'clientName', 'deviceName', + // request attributes (firewall rule matching, exact-match) + 'protocol', 'accept', 'acceptLanguage', 'queryKeys', + // ip reputation (placeholder — bounded verdict enum, exact-match) + 'ipReputation', + // premium geo (latitude/longitude are display-only: no exact-match + // filtering, so intentionally left unindexed) + 'postalCode', 'timeZone', 'weatherCode', ]; - $setIndexed = ['status', 'method', 'country', 'service', 'clientType', 'osName']; + $setIndexed = [ + 'status', 'method', 'country', 'service', 'clientType', 'osName', + // low-cardinality request/geo dims filtered by equality. accept/ + // acceptLanguage/queryKeys are unbounded caller input, so they use + // bloom_filter (below) instead of an unlimited set(0) index. + 'protocol', 'ipReputation', 'timeZone', 'weatherCode', + ]; + + // Columns whose full length exceeds the SQL adapter's max index key + // length (768 bytes) must be indexed on a prefix; ClickHouse ignores + // the prefix and indexes the whole value. + $prefixed = [ + 'path' => 255, + 'accept' => 255, + 'queryKeys' => 255, + ]; return array_map( - static function (string $col) use ($setIndexed): array { + static function (string $col) use ($setIndexed, $prefixed): array { $entry = [ '$id' => 'index-' . $col, 'type' => 'key', 'attributes' => [$col], 'indexType' => in_array($col, $setIndexed, true) ? 'set(0)' : 'bloom_filter', ]; - if ($col === 'path') { - $entry['lengths'] = [255]; + if (isset($prefixed[$col])) { + $entry['lengths'] = [$prefixed[$col]]; } return $entry; }, diff --git a/tests/Usage/Adapter/ClickHouseColumnTypeTest.php b/tests/Usage/Adapter/ClickHouseColumnTypeTest.php index b6cf638..b39377f 100644 --- a/tests/Usage/Adapter/ClickHouseColumnTypeTest.php +++ b/tests/Usage/Adapter/ClickHouseColumnTypeTest.php @@ -73,6 +73,39 @@ public function testHighCardinalityPremiumGeoColumns(): void } } + /** + * Low-cardinality request-attribute and premium-geo dims added in #36/#37 + * must map to LowCardinality(Nullable(String)). + */ + public function testLowCardinalityRequestAndGeoColumns(): void + { + foreach (['protocol', 'ipReputation', 'timeZone', 'weatherCode'] as $col) { + $this->assertSame( + 'LowCardinality(Nullable(String))', + $this->columnType($col), + "{$col} should be LowCardinality(Nullable(String))" + ); + } + } + + /** + * High-cardinality dims — un-normalized request text (accept/acceptLanguage/ + * queryKeys) and premium-geo — must fall through to plain Nullable(String). + */ + public function testHighCardinalityRequestAndGeoColumns(): void + { + foreach ([ + 'accept', 'acceptLanguage', 'queryKeys', + 'postalCode', 'latitude', 'longitude', + ] as $col) { + $this->assertSame( + 'Nullable(String)', + $this->columnType($col), + "{$col} should be plain Nullable(String)" + ); + } + } + /** * SDK dims must map to LowCardinality(Nullable(String)). */ diff --git a/tests/Usage/Adapter/ClickHouseSchemaTest.php b/tests/Usage/Adapter/ClickHouseSchemaTest.php index 1742f6a..51de694 100644 --- a/tests/Usage/Adapter/ClickHouseSchemaTest.php +++ b/tests/Usage/Adapter/ClickHouseSchemaTest.php @@ -100,6 +100,23 @@ public function testEventsTableSwapsBloomForSetOnLowCardinality(): void $this->assertStringContainsString('`index-hostname` hostname TYPE bloom_filter', $ddl); $this->assertStringContainsString('`index-resourceId` resourceId TYPE bloom_filter', $ddl); $this->assertStringContainsString('`index-teamId` teamId TYPE bloom_filter', $ddl); + + // request-attribute + premium-geo dims added for firewall/console filtering: + // bounded-cardinality equality dims get set(0); un-normalized text and + // high-cardinality dims get bloom_filter. + $this->assertStringContainsString('`index-protocol` protocol TYPE set(0)', $ddl); + $this->assertStringContainsString('`index-ipReputation` ipReputation TYPE set(0)', $ddl); + $this->assertStringContainsString('`index-timeZone` timeZone TYPE set(0)', $ddl); + $this->assertStringContainsString('`index-weatherCode` weatherCode TYPE set(0)', $ddl); + + $this->assertStringContainsString('`index-accept` accept TYPE bloom_filter', $ddl); + $this->assertStringContainsString('`index-acceptLanguage` acceptLanguage TYPE bloom_filter', $ddl); + $this->assertStringContainsString('`index-queryKeys` queryKeys TYPE bloom_filter', $ddl); + $this->assertStringContainsString('`index-postalCode` postalCode TYPE bloom_filter', $ddl); + + // latitude/longitude are display-only and intentionally NOT indexed. + $this->assertStringNotContainsString('`index-latitude`', $ddl); + $this->assertStringNotContainsString('`index-longitude`', $ddl); } public function testDailyTableMatchesPrePrSchema(): void @@ -299,8 +316,10 @@ private function expectedDimAssertions(array $columns, string $type): array 'clientEngine', 'clientEngineVersion', 'deviceName', 'deviceBrand', 'deviceModel', 'hostname', 'ip', + 'protocol', 'ipReputation', 'continentCode', 'subdivisions', 'connectionType', 'connectionUsageType', 'autonomousSystemNumber', + 'timeZone', 'weatherCode', 'sdk', 'sdkVersion', 'ordinal', ]; diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index 720d688..2146319 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -103,9 +103,16 @@ public function testEventIndexesCoverNewFilterableColumns(): void 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', 'osName', 'clientType', 'clientName', 'deviceName', + 'protocol', 'accept', 'acceptLanguage', 'queryKeys', + 'ipReputation', + 'postalCode', 'timeZone', 'weatherCode', ] as $col) { $this->assertContains($col, $indexed, "Event indexes missing {$col}"); } + + // latitude/longitude are display-only and intentionally NOT indexed. + $this->assertNotContains('latitude', $indexed); + $this->assertNotContains('longitude', $indexed); } public function testGaugeIndexesCoverIdColumns(): void @@ -610,6 +617,7 @@ public function testEventColumnsConstant(): void 'teamId', 'teamInternalId', 'country', 'region', 'hostname', 'ip', 'protocol', 'accept', 'acceptLanguage', 'queryKeys', + 'ipReputation', 'city', 'continentCode', 'subdivisions', 'postalCode', 'latitude', 'longitude', 'timeZone', 'weatherCode', 'isp', 'autonomousSystemNumber', 'autonomousSystemOrganization',