feat(clickhouse): make rollup routing reach real windows; add backfillDaily and latest-value gauge projections

src/Usage/Adapter.php

@@ -218,4 +218,19 @@ abstract public function sumDaily(string $tenant, array $queries = [], string $a
      * @return array<string, int> Metric name => sum value
      */
     abstract public function sumDailyBatch(string $tenant, array $metrics, array $queries = []): array;
+
+    /**
+     * Backfill the pre-aggregated daily events table from the raw events
+     * table for the half-open, UTC-midnight-aligned window [$from, $to).
+     *
+     * Only meaningful on adapters that maintain a daily rollup; the base
+     * refuses so a mis-wired caller fails loudly instead of silently
+     * backfilling nothing.
+     *
+     * @throws \Exception
+     */
+    public function backfillDaily(string $from, string $to, bool $force = false): void
+    {
+        throw new \Exception(static::class . ' does not maintain a daily rollup to backfill.');
+    }
 }

src/Usage/Usage.php

@@ -287,4 +287,17 @@ public function sumDailyBatch(string $tenant, array $metrics, array $queries = [
     {
         return $this->adapter->sumDailyBatch($tenant, $metrics, $queries);
     }
+
+    /**
+     * Backfill the pre-aggregated daily events table from the raw events
+     * table for the half-open, UTC-midnight-aligned window [$from, $to).
+     * Refuses a window that already holds rollup rows unless $force is
+     * passed after the caller cleared the range.
+     *
+     * @throws \Exception
+     */
+    public function backfillDaily(string $from, string $to, bool $force = false): void
+    {
+        $this->adapter->backfillDaily($from, $to, $force);
+    }
 }

tests/Usage/Adapter/ClickHouseBackfillTest.php

@@ -0,0 +1,173 @@
+<?php
+
+namespace Utopia\Tests\Adapter;
+
+use DateTime;
+use DateTimeZone;
+use Exception;
+use Utopia\Query\Query;
+use Utopia\Tests\Usage\Adapter\ClickHouseTestCase;
+use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter;
+use Utopia\Usage\Usage;
+
+class ClickHouseBackfillTest extends ClickHouseTestCase
+{
+    private Usage $usage;
+
+    private ClickHouseAdapter $adapter;
+
+    protected function setUp(): void
+    {
+        $this->adapter = new ClickHouseAdapter(
+            getenv('CLICKHOUSE_HOST') ?: 'clickhouse',
+            getenv('CLICKHOUSE_USER') ?: 'default',
+            getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse',
+            (int) (getenv('CLICKHOUSE_PORT') ?: 8123),
+            (bool) (getenv('CLICKHOUSE_SECURE') ?: false),
+            namespace: 'utopia_usage_backfill',
+            database: getenv('CLICKHOUSE_DATABASE') ?: 'default',
+            sharedTables: true,
+        );
+        $this->usage = new Usage($this->adapter);
+        $this->usage->setup();
+        $this->usage->purge('1');
+
+        $this->seedHistoricalRow('backfill.metric', 100, '-5 days');
+        $this->seedHistoricalRow('backfill.metric', 200, '-3 days');
+        $this->seedHistoricalRow('backfill.other', 40, '-4 days');
+    }
+
+    protected function tearDown(): void
+    {
+        $this->usage->purge('1');
+    }
+
+    private function seedHistoricalRow(string $metric, int $value, string $modifier): void
+    {
+        $eventsTable = $this->resolveTableName($this->adapter, 'getEventsTableName');
+        $database = $this->databaseName($this->adapter);
+
+        $time = (new DateTime($modifier, new DateTimeZone('UTC')))->format('Y-m-d H:i:s.v');
+        $id = bin2hex(random_bytes(16));
+
+        $this->queryRaw($this->adapter, sprintf(
+            "INSERT INTO `%s`.`%s` (id, metric, value, time, tenant) VALUES ('%s', '%s', %d, '%s', '1')",
+            $database,
+            $eventsTable,
+            $id,
+            addslashes($metric),
+            $value,
+            $time,
+        ));
+    }
+
+    private function truncateDaily(): void
+    {
+        $dailyTable = $this->resolveTableName($this->adapter, 'getEventsDailyTableName');
+        $database = $this->databaseName($this->adapter);
+        $this->queryRaw($this->adapter, sprintf('TRUNCATE TABLE `%s`.`%s`', $database, $dailyTable));
+    }
+
+    /**
+     * @return array{string, string} day-aligned [from, to) covering the seeds
+     */
+    private function window(): array
+    {
+        $from = (new DateTime('-7 days', new DateTimeZone('UTC')))->setTime(0, 0, 0)->format('Y-m-d H:i:s');
+        $to = (new DateTime('today', new DateTimeZone('UTC')))->format('Y-m-d H:i:s');
+
+        return [$from, $to];
+    }
+
+    /**
+     * @return array<string, int>
+     */
+    private function dailyTotals(string $from, string $to): array
+    {
+        return $this->usage->sumDailyBatch('1', ['backfill.metric', 'backfill.other'], [
+            Query::greaterThanEqual('time', $from),
+            Query::lessThan('time', $to),
+        ]);
+    }
+
+    public function testBackfillRestoresTheRollupToRawTotals(): void
+    {
+        [$from, $to] = $this->window();
+
+        // The MV rolled the seeds up on insert; wipe the rollup so the window
+        // is genuinely missing, as it is for any table that predates the MV.
+        $this->truncateDaily();
+        $this->assertSame(
+            ['backfill.metric' => 0, 'backfill.other' => 0],
+            $this->dailyTotals($from, $to),
+            'the rollup must be empty before the backfill for this test to prove anything',
+        );
+
+        $this->usage->backfillDaily($from, $to);
+
+        $this->assertSame(
+            ['backfill.metric' => 300, 'backfill.other' => 40],
+            $this->dailyTotals($from, $to),
+            'a backfilled window must re-aggregate to the exact totals the raw table holds',
+        );
+    }
+
+    public function testBackfillRefusesAWindowTheRollupAlreadyCovers(): void
+    {
+        [$from, $to] = $this->window();
+
+        // The MV already rolled the seeds up on insert.
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('double-count');
+
+        $this->usage->backfillDaily($from, $to);
+    }
+
+    public function testForceBackfillsOverAnOverlappingWindow(): void
+    {
+        [$from, $to] = $this->window();
+
+        // With force, the caller owns overlap safety; here the rollup was
+        // cleared first, so force writes the same rows the guard path would.
+        $this->truncateDaily();
+        $this->usage->backfillDaily($from, $to, force: true);
+
+        $this->assertSame(
+            ['backfill.metric' => 300, 'backfill.other' => 40],
+            $this->dailyTotals($from, $to),
+        );
+    }
+
+    public function testBackfillRejectsPartialDayBounds(): void
+    {
+        $from = (new DateTime('-7 days', new DateTimeZone('UTC')))->setTime(12, 0, 0)->format('Y-m-d H:i:s');
+        $to = (new DateTime('today', new DateTimeZone('UTC')))->format('Y-m-d H:i:s');
+
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('UTC midnights');
+
+        $this->usage->backfillDaily($from, $to);
+    }
+
+    public function testBackfillRejectsNonUtcMidnightOffsets(): void
+    {
+        // Midnight in +05:00 is 19:00 UTC — accepting it would backfill a
+        // different range than the caller intended.
+        $to = (new DateTime('today', new DateTimeZone('UTC')))->format('Y-m-d H:i:s');
+
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('UTC midnights');
+
+        $this->usage->backfillDaily('2026-07-01 00:00:00+05:00', $to);
+    }
+
+    public function testBackfillRejectsAnInvertedWindow(): void
+    {
+        [$from, $to] = $this->window();
+
+        $this->expectException(Exception::class);
+        $this->expectExceptionMessage('ascending');
+
+        $this->usage->backfillDaily($to, $from);
+    }
+}

tests/Usage/Adapter/ClickHouseGaugeProjectionTest.php

@@ -0,0 +1,135 @@
+<?php
+
+namespace Utopia\Tests\Adapter;
+
+use DateTime;
+use DateTimeZone;
+use Utopia\Query\Query;
+use Utopia\Tests\Usage\Adapter\ClickHouseTestCase;
+use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter;
+use Utopia\Usage\Usage;
+use Utopia\Usage\UsageQuery;
+
+/**
+ * The gauge projections must actually be selectable by the optimizer: the
+ * old shape carried raw `time` as a GROUP BY key, making the projection one
+ * row per sample — a copy of the base table the optimizer never used
+ * (verified in production: force_optimize_projection returned
+ * PROJECTION_NOT_USED on every grouped gauge read).
+ */
+class ClickHouseGaugeProjectionTest extends ClickHouseTestCase
+{
+    private Usage $usage;
+
+    private ClickHouseAdapter $adapter;
+
+    protected function setUp(): void
+    {
+        $this->adapter = new ClickHouseAdapter(
+            getenv('CLICKHOUSE_HOST') ?: 'clickhouse',
+            getenv('CLICKHOUSE_USER') ?: 'default',
+            getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse',
+            (int) (getenv('CLICKHOUSE_PORT') ?: 8123),
+            (bool) (getenv('CLICKHOUSE_SECURE') ?: false),
+            namespace: 'utopia_usage_gauge_proj',
+            database: getenv('CLICKHOUSE_DATABASE') ?: 'default',
+            sharedTables: true,
+        );
+        $this->usage = new Usage($this->adapter);
+
+        // A leftover table from an earlier run may carry the old projection
+        // shape under the same name (setup() adds IF NOT EXISTS); drop it so
+        // the schema under test is the one this code creates.
+        $gauges = $this->resolveTableName($this->adapter, 'getGaugesTableName');
+        $database = $this->databaseName($this->adapter);
+        $this->queryRaw($this->adapter, "DROP TABLE IF EXISTS `{$database}`.`{$gauges}`");
+
+        $this->usage->setup();
+        $this->usage->purge('1');
+
+        // Two samples per series so argMax has something to resolve, plus a
+        // second resourceType under the same metric.
+        $this->seedGaugeRow('proj.users', 5, 'user', '-3 days');
+        $this->seedGaugeRow('proj.users', 9, 'user', '-1 hour');
+        $this->seedGaugeRow('proj.users', 2, 'team', '-2 days');
+    }
+
+    protected function tearDown(): void
+    {
+        $this->usage->purge('1');
+    }
+
+    private function seedGaugeRow(string $metric, int $value, string $resourceType, string $modifier): void
+    {
+        $gauges = $this->resolveTableName($this->adapter, 'getGaugesTableName');
+        $database = $this->databaseName($this->adapter);
+        $time = (new DateTime($modifier, new DateTimeZone('UTC')))->format('Y-m-d H:i:s.v');
+        $id = bin2hex(random_bytes(16));
+
+        $this->queryRaw($this->adapter, sprintf(
+            "INSERT INTO `%s`.`%s` (id, metric, value, time, resourceType, tenant) VALUES ('%s', '%s', %d, '%s', '%s', '1')",
+            $database,
+            $gauges,
+            $id,
+            addslashes($metric),
+            $value,
+            $time,
+            addslashes($resourceType),
+        ));
+    }
+
+    public function testGroupedLatestValueReadIsServedByAProjection(): void
+    {
+        $gauges = $this->resolveTableName($this->adapter, 'getGaugesTableName');
+        $database = $this->databaseName($this->adapter);
+
+        // force_optimize_projection makes ClickHouse ERROR when no projection
+        // matches — success here IS the assertion that the shape matches.
+        $forced = $this->queryRaw($this->adapter, sprintf(
+            'SELECT `metric`, argMax(`value`, `time`) AS `value`, `resourceType`'
+            . " FROM `%s`.`%s` WHERE `tenant` IN ('1') AND `metric` IN ('proj.users')"
+            . ' GROUP BY `metric`, `resourceType`'
+            . ' SETTINGS optimize_use_projections = 1, force_optimize_projection = 1'
+            . ' FORMAT JSON',
+            $database,
+            $gauges,
+        ));
+
+        $decoded = json_decode($forced, true);
+        $rows = is_array($decoded) && is_array($decoded['data'] ?? null) ? $decoded['data'] : [];
+        $byType = [];
+        foreach ($rows as $row) {
+            if (is_array($row) && is_string($row['resourceType'] ?? null) && is_numeric($row['value'] ?? null)) {
+                $byType[$row['resourceType']] = (int) $row['value'];
+            }
+        }
+
+        $this->assertSame(
+            ['user' => 9, 'team' => 2],
+            ['user' => $byType['user'] ?? null, 'team' => $byType['team'] ?? null],
+            'the projection-served read must return the latest sample per series, same as raw argMax',
+        );
+    }
+
+    public function testAdapterGroupedReadMatchesProjectionServedValues(): void
+    {
+        // The adapter's own grouped read (the billing prefetch shape) must
+        // agree with the projection-served result.
+        $rows = $this->usage->find('1', [
+            Query::equal('metric', ['proj.users']),
+            UsageQuery::groupBy('resourceType'),
+            Query::limit(100),
+        ], Usage::TYPE_GAUGE);
+
+        $byType = [];
+        foreach ($rows as $row) {
+            $resourceType = $row->getAttribute('resourceType');
+            if (is_string($resourceType)) {
+                $byType[$resourceType] = (int) $row->getValue();
+            }
+        }
+
+        $this->assertSame(9, $byType['user'] ?? null);
+        $this->assertSame(2, $byType['team'] ?? null);
+    }
+}