diff --git a/.github/workflows/deploy-dev.js.yml b/.github/workflows/deploy-dev.js.yml index e7ee691c..baf5e1cb 100644 --- a/.github/workflows/deploy-dev.js.yml +++ b/.github/workflows/deploy-dev.js.yml @@ -64,7 +64,7 @@ jobs: - name: Reject merge conflict markers in dep files run: | - if grep -r '^<<<<<<<' package.json package-lock.json ci-deps.dev.json ci-deps.prod.json 2>/dev/null; then + if grep -r '^<<<<<<<' package.json package-lock.json ci-deps.dev.json ci-deps.prod.json crons/ci-deps.dev.json crons/ci-deps.prod.json crons/package.json 2>/dev/null; then echo "Conflict markers found in dependency files" exit 1 fi @@ -79,8 +79,9 @@ jobs: env: AP_GAMESLIB_VERSION: ${{ github.event.client_payload.gameslib_version }} AP_RENDERER_VERSION: ${{ github.event.client_payload.renderer_version }} + AP_RECRANKS_VERSION: ${{ github.event.client_payload.recranks_version }} AP_SOURCE: ${{ github.event.client_payload.source_run_id && format('gameslib-ci-{0}', github.event.client_payload.source_run_id) || '' }} - run: npx ap-install-deps --stage dev + run: npm run sync-deps - name: Verify lockfile matches ci-deps run: npx ap-check-ci-deps --stage dev --strict @@ -94,6 +95,8 @@ jobs: ci-deps.dev.json package-lock.json package.json + crons/ci-deps.dev.json + crons/package.json # --- MACHINE TRANSLATION PIPELINE START --- - name: Prune stale managed locale keys @@ -131,6 +134,12 @@ jobs: - name: Test (Lambda init smoke) run: npm test + - name: Test crons + run: npm run test:crons + + - name: Test crons Lambda layers + run: npm run test:crons:layers + - name: Configure AWS credentials uses: Fooji/create-aws-profile-action@v1 with: @@ -144,9 +153,12 @@ jobs: # aws-secret-access-key: ${{ secrets.AWS_SECRET }} # aws-region: us-east-1 - - name: Deploy + - name: Deploy API stack run: bash bin/serverless-deploy.sh dev AbstractPlayDev + - name: Deploy crons stack + run: bash crons/bin/serverless-deploy.sh dev + - name: Trigger docs rebuild if: github.event_name == 'push' run: | @@ -162,7 +174,7 @@ jobs: git fetch --depth=1 origin "$BEFORE" CHANGED="$(git diff --name-only "$BEFORE" "$SHA")" fi - if echo "$CHANGED" | grep -q '^docs/'; then + if echo "$CHANGED" | grep -qE '^(docs/|crons/docs/)'; then curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ diff --git a/.github/workflows/deploy-prod.js.yml b/.github/workflows/deploy-prod.js.yml index a10c2945..d1ade9fe 100644 --- a/.github/workflows/deploy-prod.js.yml +++ b/.github/workflows/deploy-prod.js.yml @@ -63,7 +63,7 @@ jobs: - name: Reject merge conflict markers in dep files run: | - if grep -r '^<<<<<<<' package.json package-lock.json ci-deps.dev.json ci-deps.prod.json 2>/dev/null; then + if grep -r '^<<<<<<<' package.json package-lock.json ci-deps.dev.json ci-deps.prod.json crons/ci-deps.dev.json crons/ci-deps.prod.json crons/package.json 2>/dev/null; then echo "Conflict markers found in dependency files" exit 1 fi @@ -78,8 +78,9 @@ jobs: env: AP_GAMESLIB_VERSION: ${{ github.event.client_payload.gameslib_version }} AP_RENDERER_VERSION: ${{ github.event.client_payload.renderer_version }} + AP_RECRANKS_VERSION: ${{ github.event.client_payload.recranks_version }} AP_SOURCE: ${{ github.event.client_payload.source_run_id && format('gameslib-ci-{0}', github.event.client_payload.source_run_id) || '' }} - run: npx ap-install-deps --stage prod + run: npm run sync-deps:prod - name: Verify lockfile matches ci-deps run: npx ap-check-ci-deps --stage prod --strict @@ -93,6 +94,8 @@ jobs: ci-deps.prod.json package-lock.json package.json + crons/ci-deps.prod.json + crons/package.json - name: Set CI version run: npm version prerelease --preid=ci-$GITHUB_RUN_ID --no-git-tag-version @@ -105,6 +108,12 @@ jobs: - name: Test (Lambda init smoke) run: npm test + - name: Test crons + run: npm run test:crons + + - name: Test crons Lambda layers + run: npm run test:crons:layers + - name: Configure AWS credentials uses: Fooji/create-aws-profile-action@v1 with: @@ -118,9 +127,12 @@ jobs: # aws-secret-access-key: ${{ secrets.AWS_SECRET }} # aws-region: us-east-1 - - name: Deploy + - name: Deploy API stack run: bash bin/serverless-deploy.sh prod AbstractPlayProd + - name: Deploy crons stack + run: bash crons/bin/serverless-deploy.sh prod + - name: Trigger docs rebuild if: github.event_name == 'push' run: | @@ -136,7 +148,7 @@ jobs: git fetch --depth=1 origin "$BEFORE" CHANGED="$(git diff --name-only "$BEFORE" "$SHA")" fi - if echo "$CHANGED" | grep -q '^docs/'; then + if echo "$CHANGED" | grep -qE '^(docs/|crons/docs/)'; then curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a473e826..c74ef10c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: - name: Reject merge conflict markers in dep files run: | - if grep -r '^<<<<<<<' package.json package-lock.json ci-deps.dev.json ci-deps.prod.json 2>/dev/null; then + if grep -r '^<<<<<<<' package.json package-lock.json ci-deps.dev.json ci-deps.prod.json crons/ci-deps.dev.json crons/ci-deps.prod.json crons/package.json 2>/dev/null; then echo "Conflict markers found in dependency files" exit 1 fi @@ -44,6 +44,9 @@ jobs: - name: Install pinned AP dependencies run: npx ap-install-deps --stage dev + - name: Sync crons AP manifests from lockfile + run: node scripts/sync-crons-ap-deps.mjs --stage dev + - name: Verify lockfile matches ci-deps run: npx ap-check-ci-deps --stage dev --strict @@ -54,3 +57,37 @@ jobs: - name: Test (Lambda init smoke) run: npm test + + test-crons: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Pin npm + run: npm install -g npm@11.6.2 + + - name: Create .npmrc + run: echo "@abstractplay:registry=https://npm.pkg.github.com/" > .npmrc + - run: echo "//npm.pkg.github.com/:_authToken=${{secrets.PAT_READ_PACKAGES}}" >> .npmrc + + - name: Install NPM dependencies + run: npm ci + + - name: Install pinned AP dependencies + run: npx ap-install-deps --stage dev + + - name: Sync crons AP manifests from lockfile + run: node scripts/sync-crons-ap-deps.mjs --stage dev + + - name: Lint crons + run: npm run lint:crons + + - name: Test crons + run: npm run test:crons + + - name: Test crons Lambda layers + run: npm run test:crons:layers diff --git a/.gitignore b/.gitignore index 5a76ebc1..1765be2a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ jspm_packages # Serverless directories .serverless +crons/.serverless +crons/.esbuild .esbuild/ .test-artifacts/ dist/ diff --git a/ci-deps.dev.json b/ci-deps.dev.json index c9e31831..0a3ac79a 100644 --- a/ci-deps.dev.json +++ b/ci-deps.dev.json @@ -1,6 +1,6 @@ { "renderer": "1.0.0-ci-35278756813.0", - "updatedAt": "2026-09-19T16:00:34.835Z", - "source": "gameslib-ci-35453221145", + "updatedAt": "2026-09-19T16:06:26.073Z", + "source": "ci-deps.dev.json", "gameslib": "1.0.0-ci-35453221145.0" } diff --git a/crons/.eslintrc.json b/crons/.eslintrc.json new file mode 100644 index 00000000..18d83775 --- /dev/null +++ b/crons/.eslintrc.json @@ -0,0 +1,39 @@ +{ + "env": { + "browser": true, + "commonjs": true, + "es2021": true + }, + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended" + ], + "ignorePatterns": [ + "scripts/index.html" + ], + "overrides": [ + { + "files": ["**/*.ts"], + "rules": { + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/no-non-null-asserted-optional-chain": "off", + "@typescript-eslint/no-unused-vars": "warn", + "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-extra-semi": "off" + } + } + ], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "project": ["./tsconfig.json"] + }, + "plugins": [ + "@typescript-eslint" + ], + "rules": { + "@typescript-eslint/await-thenable": "error", + "@typescript-eslint/no-floating-promises": ["error"] + } +} diff --git a/crons/.gitattributes b/crons/.gitattributes new file mode 100644 index 00000000..53ef5e0b --- /dev/null +++ b/crons/.gitattributes @@ -0,0 +1,8 @@ +# Keep production dependency pins when merging develop into main. +ci-deps.prod.json merge=ours + +# Keep develop dependency pins when merging l10n/weblate (or other branches) into develop. +ci-deps.dev.json merge=ours + +# Keep target-branch lockfile on cross-branch merges; run sync-deps if strict check fails. +package-lock.json merge=ours diff --git a/crons/.gitignore b/crons/.gitignore new file mode 100644 index 00000000..3b9151ed --- /dev/null +++ b/crons/.gitignore @@ -0,0 +1,15 @@ +# package directories +node_modules +jspm_packages + +# Serverless directories +.serverless +/api/*.js +/utils/*.js +.vscode/ +bin/* +dist/ + +# Generated from node-backend (see src/locales/README.md) +src/locales/** +!src/locales/README.md diff --git a/crons/.nvmrc b/crons/.nvmrc new file mode 100644 index 00000000..a45fd52c --- /dev/null +++ b/crons/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/crons/LICENSE b/crons/LICENSE new file mode 100644 index 00000000..1a60652c --- /dev/null +++ b/crons/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Abstract Play + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crons/README.md b/crons/README.md new file mode 100644 index 00000000..15ed6f8d --- /dev/null +++ b/crons/README.md @@ -0,0 +1,7 @@ +# Backend Crons + +This tree lives in the [node-backend](https://github.com/AbstractPlay/node-backend) monorepo (`crons/`). Deploy with `serverless` from this directory after the main API stack (see `docs/deployment.md`). + +Scheduled AWS Lambda jobs for Abstract Play: DynamoDB exports, static game records on S3, site-wide analytics, and live tournament/challenge automation. + +Developer documentation: [Abstract Play docs — Crons](https://docs.abstractplay.com/crons/) (dev: `docs.dev.abstractplay.com`). diff --git a/crons/ci-deps.dev.json b/crons/ci-deps.dev.json new file mode 100644 index 00000000..7226e423 --- /dev/null +++ b/crons/ci-deps.dev.json @@ -0,0 +1,7 @@ +{ + "updatedAt": "2026-09-19T16:07:51.415Z", + "source": "ci-deps.dev.json", + "renderer": "1.0.0-ci-35278756813.0", + "gameslib": "1.0.0-ci-35453221145.0", + "recranks": "1.0.0-ci-35280155165.0" +} diff --git a/crons/ci-deps.prod.json b/crons/ci-deps.prod.json new file mode 100644 index 00000000..32837b9f --- /dev/null +++ b/crons/ci-deps.prod.json @@ -0,0 +1,7 @@ +{ + "renderer": "1.0.0-ci-32782926153.0", + "updatedAt": "2026-08-24T22:13:46.262Z", + "source": "gameslib-ci-32783108998", + "gameslib": "1.0.0-ci-32783108998.0", + "recranks": "1.0.0-ci-32677322618.0" +} diff --git a/crons/docs/_docs-repo-integration.md b/crons/docs/_docs-repo-integration.md new file mode 100644 index 00000000..807d04b5 --- /dev/null +++ b/crons/docs/_docs-repo-integration.md @@ -0,0 +1,14 @@ +# Docs repository integration + +[AbstractPlay/docs](https://github.com/AbstractPlay/docs) aggregates crons documentation from this monorepo: + +- Submodule `vendor/node-backend` → `https://github.com/AbstractPlay/node-backend.git` (`develop` / `main`). +- Prebuild: `vendor/node-backend/crons/docs` → site prefix `/crons/` (`scripts/crons-docs.js` + `syncDocsFromSrc` in `scripts/prebuild.js`). +- Site nav: **Crons** section at `/crons/` (`crons/docs/nav.json`). +- No `vendor/backend-crons` submodule (retired with monorepo merge). + +Local prebuild: sibling `../node-backend` with `crons/docs/` when the vendor pin predates the merge. + +Published URL prefix: `/crons/` (e.g. `/crons/pipeline/`). + +Docs rebuild trigger: node-backend deploy workflow dispatches `dep_update_dev` / `dep_update_prod` when a push changes `docs/` or `crons/docs/`. diff --git a/crons/docs/architecture.md b/crons/docs/architecture.md new file mode 100644 index 00000000..ee461fd6 --- /dev/null +++ b/crons/docs/architecture.md @@ -0,0 +1,101 @@ +# Architecture + +## Overview + +[backend-crons](https://github.com/AbstractPlay/backend-crons) is a Serverless Framework v3 service (`abstract-play-backend-crons`) deployed to AWS `us-east-1`. All functions are Node.js 20 Lambdas triggered by EventBridge cron rules (prod only). + +There is no API Gateway — these are batch and maintenance jobs only. + +## Lambda functions + +Defined in [`serverless.yml`](../serverless.yml). Schedules are **daily** unless noted (EventBridge `cron(0 H * * ? *)` = every day at hour H UTC). + +| Function | Schedule (UTC, prod) | Role | +|----------|----------------------|------| +| `dumpdb` | Daily 00:00 | Export prod DynamoDB to S3 | +| `records` | Daily 03:00 | Build game records from dump | +| `records-ttm` | Daily 03:00 | Per-player time-to-move arrays | +| `records-move-times` | Daily 03:00 | Move activity histograms | +| `records-cooccur` | Daily 03:00 | PMI co-occurrence for recommendations | +| `records-rec-analytics` | Daily 03:00 | Recommendation impression funnel analytics (ops S3) | +| `tournament-data` | Daily 03:00 | Tournament summaries from dump | +| `records-manifest` | Daily 04:00 and 07:30 | S3 listing + `_manifest.json` v2 | +| `summarize` | Daily 06:00 | Site analytics from `ALL.json` | +| `player-summary-fanout` | Daily 06:15 | SQS fan-out for `player/*-summary.json` | +| `player-summary-worker` | SQS-triggered | Writes one player summary slice per message | +| `starttournaments` | Daily 10:00 and 22:00 | Start/cancel tournaments, create games | +| `standingchallenges` | Daily 00:00 and 12:00 | Process preset standing challenges | + +See [Records pipeline](/crons/pipeline/) and [Functions reference](/crons/functions/) for details. + +## Gameslib Lambda layer + +Functions that call `@abstractplay/gameslib` attach the `abstractplayGameslib` layer, built by [`scripts/build-layers.mjs`](https://github.com/AbstractPlay/backend-crons/blob/develop/scripts/build-layers.mjs) before packaging: + +- Bundles `@abstractplay/gameslib` and `@abstractplay/recranks` into `.serverless/layers/abstractplay-gameslib` +- Strips `@abstractplay/renderer` (transitive dep, not needed at runtime) +- Prunes docs and tests from the layer to stay under Lambda size limits; retains gameslib `locales/en/` for variant name resolution during record generation + +esbuild marks `@abstractplay/gameslib` and `@abstractplay/recranks` as **external** so they resolve from the layer at runtime, not from the function bundle. + +## Data flow + +```mermaid +flowchart LR + ddb[(DynamoDB abstract-play-prod)] --> dumpdb + dumpdb --> s3dump[(S3 abstractplay-db-dump)] + s3dump --> records + s3dump --> cooccur[records-cooccur] + records --> s3rec[(S3 records.abstractplay.com)] + cooccur --> s3rec + s3rec --> summarize + summarize --> s3rec + ddb --> live[Live crons] + ddb --> recanalytics[records-rec-analytics] + recanalytics --> s3ops[(private ops S3)] + live --> ddb + s3rec --> cf[CloudFront CDN] +``` + +## IAM permissions + +The service role grants: + +- **DynamoDB** — query/scan/get/put/update/delete, batch write, point-in-time export +- **S3** — list/get on `abstractplay-db-dump`; put on `records.abstractplay.com` and dump bucket; get/put on private ops bucket (`recommendations/analytics/*`) +- **SES** — send email (tournament notifications) +- **CloudFront** — gzip + cache headers on records CDN (see [S3 outputs](/crons/s3-outputs/#cloudfront-and-caching)); no blanket invalidation + +## Environment + +| Variable | Source | Purpose | +|----------|--------|---------| +| `ABSTRACT_PLAY_TABLE` | `serverless.yml` | `abstract-play-{stage}` — used by `summarize` (geo stats), live crons, and `records-rec-analytics` | + +## Dependencies + +| Package | Used by | Purpose | +|---------|---------|---------| +| `@abstractplay/gameslib` | records, records-ttm, records-move-times, summarize, player-summary-fanout, rating-change-notifications, starttournaments | `GameFactory`, `gameinfo`, `genRecord`, `addResource` | +| `@abstractplay/recranks` | records, summarize | `APGameRecord`, ELO/Glicko2/Trueskill raters | +| `ion-js`, `fflate` | dump consumers | Parse gzipped ION export files | +| `i18next` | starttournaments, inactive-challenge-cleanup, records/thumbnails (gameslib labels) | Vendored `apback` email/push copy; gameslib bundles for meta-game names | + +`records-cooccur` uses `ion-js` and `fflate` only (no gameslib layer). + +## Project layout + +``` +src/functions/ Lambda handlers +src/types/ Record and StatSummary TypeScript types +src/locales/ Generated apback JSON (CI/local sync from node-backend; not committed) +src/utils/ Shared utilities (e.g. isoToCountryCode, cooccurPmi) +scripts/ build-layers.mjs, locale sync/check, operator helpers +serverless.yml Infrastructure and schedules +``` + +## Related + +- [Getting started](/crons/getting-started/) +- [Deployment](/crons/deployment/) +- [Backend architecture](/backend/architecture/) diff --git a/crons/docs/dashboard-cruft-cleanup.md b/crons/docs/dashboard-cruft-cleanup.md new file mode 100644 index 00000000..d6998d73 --- /dev/null +++ b/crons/docs/dashboard-cruft-cleanup.md @@ -0,0 +1,64 @@ +# Dashboard cruft cleanup + +Daily cron that prunes stale **index-only** dashboard rows for users inactive ≥ 1 year. Complements lazy `me()` eviction for active users and one-off ops scripts (`prune-stale-recent-completed`, `purge-usergame-orphans`). + +## Schedule + +| | | +|---|---| +| **Handler** | [`src/functions/dashboard-cruft-cleanup.ts`](../src/functions/dashboard-cruft-cleanup.ts) | +| **Schedule** | Daily 03:00 UTC (prod only) | +| **Timeout / memory** | 900 s / 1024 MB | +| **Layer** | No | + +## What it cleans + +For each eligible user (live confirm after S3 dump candidate scan): + +| Partition | Action | +|-----------|--------| +| `RECENTCOMPLETED#` | Delete rows not dashboard-eligible (merged `USERGAME#` overlays) | +| `USERGAME#` | Delete overlays for pruned recent rows and orphan overlays not on `CURRENTGAMES#` ∪ eligible `RECENTCOMPLETED#` | + +**Does not** touch `USER.lastSeen`, `USERS.lastSeen`, or legacy `USER.games[]` (retired in Phase 5). + +## Candidate discovery (no DynamoDB scan) + +1. List `abstractplay-db-dump` and pick the latest `manifest-summary.json` export uid (same pattern as [Records pipeline](/crons/pipeline/)). +2. Stream ION `USER` items from that export. +3. Collect `sk` (user id) when `lastSeen < now - 1y` and `cleaned != true`. + +Dump may be up to ~24h stale; each candidate is re-validated with live `GetItem` before cleanup. + +## Live processing + +For up to `DASHBOARD_CRUFT_BATCH_SIZE` candidates (default 75): + +1. `GetItem` `USER` — skip if active since dump, already `cleaned`, or missing. +2. Skip bots (`GetItem` `BOT`). +3. Run [`cleanupUserDashboardCruft`](../src/utils/dashboardCruftCleanup.ts) (Query + Delete only). +4. If cruft was removed, `SET cleaned = true` on `USER` (not `USERS`). + +`me()` in node-backend clears `cleaned` on login so a future long absence can be cleaned again. + +## Environment + +| Variable | Default | Purpose | +|----------|---------|---------| +| `ABSTRACT_PLAY_TABLE` | `abstract-play-{stage}` | DynamoDB table | +| `DASHBOARD_CRUFT_BATCH_SIZE` | `75` | Max users processed per run | +| `ABANDONED_ACCOUNT_INACTIVE_MS` | `31536000000` (365 days) | Inactivity threshold | + +## Manual invoke + +```bash +serverless invoke -f dashboard-cruft-cleanup --stage prod +``` + +Use prod with care — this deletes dashboard index rows for inactive users. + +## Related + +- [Live crons](/crons/live-crons/) — other live DynamoDB mutators +- [Backend database schema](/backend/database-schema/) — `USER.cleaned` +- node-backend `lib/dashboardCruftCleanup.ts` — shared cleanup logic (keep in sync) diff --git a/crons/docs/deployment.md b/crons/docs/deployment.md new file mode 100644 index 00000000..cb14b48d --- /dev/null +++ b/crons/docs/deployment.md @@ -0,0 +1,66 @@ +# Deployment + +Crons deploy from the [node-backend](https://github.com/AbstractPlay/node-backend) monorepo (`crons/` directory). CI and manual deploys always run the **API stack first**, then this stack. + +## Automatic deploys + +Same GitHub Actions workflows as the API ([`deploy-dev.js.yml`](../../.github/workflows/deploy-dev.js.yml), [`deploy-prod.js.yml`](../../.github/workflows/deploy-prod.js.yml)): + +| Step | Command | +|------|---------| +| API | `bash bin/serverless-deploy.sh ` | +| Crons | `bash crons/bin/serverless-deploy.sh ` | + +| Branch / trigger | Stage | +|------------------|-------| +| `develop` push | `dev` | +| `main` push | `prod` | +| `repository_dispatch` `dep_update_*` | matching stage | + +Gameslib (and similar) should dispatch **`dep_update_*` only to node-backend** — both stacks redeploy from one workflow. + +PR CI: [`.github/workflows/test.yml`](../../.github/workflows/test.yml) job **`test-crons`** runs `lint:crons`, `test:crons`, and `test:crons:layers`. + +## AP dependency pins + +Pins live in **`crons/ci-deps.dev.json`** and **`crons/ci-deps.prod.json`**, kept in sync with the root lockfile by `node scripts/sync-crons-ap-deps.mjs` (chained from root `npm run sync-deps` / `npm run sync-deps:prod`). + +Do not run `ap-install-deps` from `crons/` alone in a workspace checkout — use root `npm run sync-deps`. + +## Manual deploy + +From repo root (after API deploy): + +```bash +npm run build -w abstractplay-backend-crons # eslint in crons/ +npm run test:crons:layers # optional but recommended +bash crons/bin/serverless-deploy.sh dev # or prod +``` + +Or from `crons/`: + +```bash +cd crons +npm run build +npm run test:layers +npx serverless deploy --stage dev +``` + +AWS profile comes from `params` in [`serverless.yml`](../serverless.yml) (`AbstractPlayDev` / `AbstractPlayProd`). + +## Schedules + +EventBridge cron rules are **enabled only on prod** (`custom.scheduleEnabled.prod: true`). Dev stacks contain the Lambdas but scheduled invocations are off. + +## Ops alerts (email) + +On **prod**, CloudWatch alarms in this stack publish to the SNS topic exported by the API stack (`abstract-play-prod-OpsAlertsTopicArn`). Confirm the ops-alerts email subscription via [node-backend deployment](/backend/deployment/#ops-alerts-email). + +## Documentation site + +Cron docs live in `crons/docs/` and publish under `/crons/` on the [docs site](https://docs.abstractplay.com). The [AbstractPlay/docs](https://github.com/AbstractPlay/docs) prebuild syncs `vendor/node-backend/crons/docs` (no separate `backend-crons` submodule). Maintainer checklist: [`_docs-repo-integration.md`](https://github.com/AbstractPlay/node-backend/blob/develop/crons/docs/_docs-repo-integration.md). + +## Related + +- [Pipeline](/crons/pipeline/) +- [Backend deployment](/backend/deployment/) diff --git a/crons/docs/functions.md b/crons/docs/functions.md new file mode 100644 index 00000000..56026074 --- /dev/null +++ b/crons/docs/functions.md @@ -0,0 +1,190 @@ +# Functions reference + +All handlers live in [`src/functions/`](https://github.com/AbstractPlay/backend-crons/tree/develop/src/functions). Schedules and resource limits are in [`serverless.yml`](../serverless.yml). + +Batch dump consumers run **daily at 03:00 UTC** and read the latest completed ION export (see [Records pipeline](/crons/pipeline/)). + +## Batch functions (S3 / dump) + +### `dumpdb` + +| | | +|---|---| +| **Handler** | `src/functions/dumpdb.ts` | +| **Schedule** | Daily 00:00 UTC | +| **Timeout / memory** | 1024 MB (default) | +| **Layer** | No | +| **Input** | EventBridge event (unused) | +| **Output** | DynamoDB export to `abstractplay-db-dump` (ION) | +| **Notes** | Exports `abstract-play-prod` table only; prunes exports older than 7 days | + +### `records` + +| | | +|---|---| +| **Handler** | `src/functions/records.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 10240 MB | +| **Layer** | gameslib | +| **Input** | Latest ION dump | +| **Output** | `ALL.json`, `meta/*.json`, `player/*.json`, `event/*.json` | +| **Notes** | Uses `GameFactory`, `genRecord`, `addResource`; marks AI players via BOT records | + +### `records-ttm` + +| | | +|---|---| +| **Handler** | `src/functions/records-ttm.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 10240 MB | +| **Layer** | gameslib | +| **Input** | Latest ION dump (GAME records) | +| **Output** | `ttm/{playerId}.json` — array of move-to-move durations (ms) | + +### `records-move-times` + +| | | +|---|---| +| **Handler** | `src/functions/records-move-times.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 10240 MB | +| **Layer** | gameslib | +| **Input** | Latest ION dump (GAME + MOVE records) | +| **Output** | `mvtimes.json` — activity histograms by meta game plus move-time seasonality | + +### `records-cooccur` + +| | | +|---|---| +| **Handler** | `src/functions/records-cooccur.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 10240 MB | +| **Layer** | No | +| **Input** | Latest ION dump (completed GAME + USER `stars[]`) | +| **Output** | `recommendations/cooccur.json` — PMI co-occurrence matrix | +| **Notes** | See [Recommendation co-occurrence](/crons/recommendations-cooccur/) | + +### `records-rec-analytics` + +| | | +|---|---| +| **Handler** | `src/functions/records-rec-analytics.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 1024 MB | +| **Layer** | No | +| **Input** | Live DynamoDB scan (`RECOMMENDS#*`) | +| **Output** | `recommendations/analytics/*` on private ops S3 | +| **Notes** | See [Recommendation analytics](/crons/recommendations-analytics/) | + +### `tournament-data` + +| | | +|---|---| +| **Handler** | `src/functions/tournament-data.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 10240 MB | +| **Layer** | No | +| **Input** | Latest ION dump (TOURNAMENT / COMPLETEDTOURNAMENT records) | +| **Output** | `tournament-summary.json`, `player/tournaments/{playerId}.json` | + +### `records-manifest` + +| | | +|---|---| +| **Handler** | `src/functions/records-manifest.ts` | +| **Schedule** | Daily 04:00 and 07:30 UTC | +| **Timeout / memory** | 900 s / 10240 MB | +| **Layer** | gameslib (attached but no gameslib import) | +| **Input** | S3 list on records bucket | +| **Output** | `_manifest.json` (v2: `summaryFiles` + `objects`); `Cache-Control: no-cache` | + +### `summarize` + +| | | +|---|---| +| **Handler** | `src/functions/summarize.ts` | +| **Schedule** | Daily 06:00 UTC | +| **Timeout / memory** | 900 s / 5120 MB | +| **Layer** | gameslib | +| **Input** | `ALL.json`; `mvtimes.json` (seasonality); live `USERS` query for geo stats | +| **Output** | `_summary.json`, `_summary-site.json`, `_summary-players.json`, `_summary-ratings.json`; `stats/rivalries.json` on private ops bucket | +| **Notes** | See [Summarize](/crons/summarize/) | + +### `player-summary-fanout` + +| | | +|---|---| +| **Handler** | `src/functions/player-summary-fanout.ts` | +| **Schedule** | Daily 06:15 UTC | +| **Timeout / memory** | 300 s / 1024 MB | +| **Layer** | gameslib | +| **Input** | `_summary-site.json`, `_summary-players.json`, `_summary-ratings.json`; previous `_summary-player-manifest.json` (optional) | +| **Output** | SQS messages (changed player slices only); `_summary-player-manifest.json` v2 | +| **Return** | `candidateCount`, `enqueuedCount`, `skippedCount`, `inputUnchanged`, `tierBytesLoaded`, `manifestBytes` | + +### `player-summary-worker` + +| | | +|---|---| +| **Handler** | `src/functions/player-summary-worker.ts` | +| **Trigger** | SQS (`PlayerSummaryQueue`, batch 5) | +| **Timeout / memory** | 30 s / 256 MB | +| **Concurrency** | 25 reserved | +| **Output** | `player/{userId}-summary.json` | + +## Live functions (DynamoDB) + +### `starttournaments` + +| | | +|---|---| +| **Handler** | `src/functions/starttournaments.ts` | +| **Schedule** | Daily 10:00 and 22:00 UTC | +| **Timeout / memory** | 600 s / 1024 MB | +| **Layer** | gameslib | +| **Input** | `TOURNAMENT` records in DynamoDB | +| **Output** | Creates/cancels tournaments, starts games via `GameFactory`, sends SES emails | +| **Notes** | See [Live crons](/crons/live-crons/) | + +### `inactive-challenge-cleanup` + +| | | +|---|---| +| **Handler** | `src/functions/inactive-challenge-cleanup.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 300 s / 512 MB | +| **Layer** | gameslib | +| **Input** | Live `USERS`, `CHALLENGE`, `STANDINGCHALLENGE#`, `METAGAMES#` | +| **Output** | Revokes stale challenges, pauses `REALSTANDING`, acceptor notifications | +| **Notes** | See [Inactive challenge cleanup](/crons/inactive-challenge-cleanup/) | + +### `standingchallenges` + +| | | +|---|---| +| **Handler** | `src/functions/standingchallenges.ts` | +| **Schedule** | Daily 00:00 and 12:00 UTC | +| **Timeout / memory** | 600 s / 1024 MB | +| **Layer** | No | +| **Input** | `REALSTANDING` preset records | +| **Output** | Issues standing challenge requests in DynamoDB | +| **Notes** | See [Live crons](/crons/live-crons/) | + +### `dashboard-cruft-cleanup` + +| | | +|---|---| +| **Handler** | `src/functions/dashboard-cruft-cleanup.ts` | +| **Schedule** | Daily 03:00 UTC | +| **Timeout / memory** | 900 s / 1024 MB | +| **Layer** | No | +| **Input** | Latest ION dump (`USER` candidates) + live DynamoDB confirm | +| **Output** | Deletes stale `RECENTCOMPLETED#` / orphan `USERGAME#`; sets `USER.cleaned` | +| **Notes** | See [Dashboard cruft cleanup](/crons/dashboard-cruft-cleanup/) | + +## Related + +- [Records pipeline](/crons/pipeline/) +- [S3 outputs](/crons/s3-outputs/) +- [Recommendation co-occurrence](/crons/recommendations-cooccur/) +- [Architecture](/crons/architecture/) diff --git a/crons/docs/getting-started.md b/crons/docs/getting-started.md new file mode 100644 index 00000000..45b2959f --- /dev/null +++ b/crons/docs/getting-started.md @@ -0,0 +1,86 @@ +# Getting started + +## Prerequisites + +- **Node.js 24** (matches `serverless.yml` runtime) +- **AWS CLI** with profiles `AbstractPlayDev` and `AbstractPlayProd` in `~/.aws/credentials` +- Access to the `@abstractplay` GitHub Packages scope + +## Install and build + +```bash +npm install +npm run build:layers # required before deploy — builds gameslib Lambda layer +npm run build # ESLint +npm test # vitest (summarizeHelpers unit tests) +``` + +## GitHub Packages + +Private packages require a `.npmrc`: + +``` +@abstractplay:registry=https://npm.pkg.github.com/ +//npm.pkg.github.com/:_authToken= +``` + +CI creates this from the `PAT_READ_PACKAGES` secret (see [`.github/workflows/deploy-dev.js.yml`](../.github/workflows/deploy-dev.js.yml)). + +## Local gameslib development + +To test against a local rules engine build: + +```bash +npm install /path/to/gameslib.tgz +npm run build:layers +``` + +Or pin the dev tag (as CI does on `develop`): + +```bash +npm i @abstractplay/gameslib@development +npm run build:layers +``` + +## Invoking a function locally + +With AWS credentials configured for the target stage: + +```bash +serverless invoke -f summarize --stage prod +serverless invoke -f records --stage prod +``` + +Most batch functions expect prod S3 buckets and a completed DB dump. For code changes, prefer unit tests (`src/functions/summarizeHelpers.test.ts`) or invoke against dev stacks with caution — schedules are disabled on dev. + +## Email strings (`apback`) + +Canonical copy lives in [node-backend `locales/`](https://github.com/AbstractPlay/node-backend/tree/develop/locales) (Weblate). Lambdas import `src/locales/*/apback.json`, but those files are **not committed** (like generated assets elsewhere in the monorepo workflow). + +**Local:** clone node-backend as a sibling (`../node-backend`) or set `NODE_BACKEND_ROOT`, then: + +```bash +npm run sync-apback-locales +``` + +`npm test` runs `pretest`, which syncs automatically when `src/locales/` is missing. + +**CI / deploy:** workflows check out node-backend and run `sync-apback-locales` before build, test, and Serverless deploy. Do not open translation-only PRs here. + +## Project layout + +``` +src/functions/ Lambda handlers (one file per function) +src/types/ Shared TypeScript types +src/locales/ Generated apback JSON (see README there; sync from node-backend) +scripts/ Repo tooling (layers build, locale sync, ops helpers) +bin/ Local-only ops scripts (gitignored; not in CI) +serverless.yml Function definitions and schedules +docs/ Developer documentation (published at /crons/) +``` + +## Next steps + +- [Architecture](/crons/architecture/) +- [Deployment](/crons/deployment/) +- [Records pipeline](/crons/pipeline/) diff --git a/crons/docs/inactive-challenge-cleanup.md b/crons/docs/inactive-challenge-cleanup.md new file mode 100644 index 00000000..0ecc490f --- /dev/null +++ b/crons/docs/inactive-challenge-cleanup.md @@ -0,0 +1,60 @@ +# Inactive challenge cleanup + +Nightly live-DynamoDB cron that revokes open and direct challenges issued by players inactive ≥ 14 days, pauses matching `REALSTANDING` presets, and notifies acceptors. + +## Schedule + +| | | +|---|---| +| **Handler** | [`src/functions/inactive-challenge-cleanup.ts`](../src/functions/inactive-challenge-cleanup.ts) | +| **Schedule** | Daily 03:00 UTC (prod only) | +| **Timeout / memory** | 300 s / 512 MB | +| **Layer** | gameslib (metaGame UID list for `METAGAMES#` scan) | + +## Discovery (challenge-centric) + +1. Query `pk=USERS` (`sk`, `lastSeen`) and build `inactiveSet` (`lastSeen < now - 14d`; missing `lastSeen` → active). +2. Query `pk=CHALLENGE` (all pending direct challenges). +3. `BatchGet` `METAGAMES#{metaGame}` counts; query `STANDINGCHALLENGE#{metaGame}` only where `standingchallenges > 0`. +4. Filter challenges where `challenger.id ∈ inactiveSet`. + +No S3 dump scan; cost scales with open challenge count, not inactive user count. + +## Actions per candidate + +1. Re-check issuer `USERS.lastSeen` (skip if logged in since discovery). +2. Revoke challenge (mirror node-backend `removeAChallenge` revocation path). +3. If standing open challenge matches a `REALSTANDING` preset entry, set `suspended: true` on that entry. +4. Notify **acceptors only** (email if prefs, push, in-app for direct challenges). + +## Environment + +| Variable | Default | Purpose | +|----------|---------|---------| +| `ABSTRACT_PLAY_TABLE` | `abstract-play-{stage}` | DynamoDB table | +| `INACTIVE_CHALLENGE_MS` | `1209600000` (14d) | Inactivity threshold | +| `INACTIVE_CHALLENGE_REVOKE_BATCH_SIZE` | unlimited | Max revokes per run (initial rollout) | +| `VAPID_PUBLIC_KEY` / `VAPID_PRIVATE_KEY` | — | Web push (optional; skipped if unset) | + +## Preview (read-only, local) + +Before the first prod run, preview candidates without writes: + +```bash +npm run preview-inactive-challenges -- --stage prod +``` + +Uses AWS profile `AbstractPlayProd` / table `abstract-play-prod`. Optional `--days 14` (default). + +## Manual invoke + +```bash +serverless invoke -f inactive-challenge-cleanup --stage prod +``` + +Use prod with care — this revokes live challenges. + +## Related + +- [Live crons](/crons/live-crons/) +- [Challenges subsystem](/backend/subsystems/challenges/) diff --git a/crons/docs/index.md b/crons/docs/index.md new file mode 100644 index 00000000..8c0fc390 --- /dev/null +++ b/crons/docs/index.md @@ -0,0 +1,37 @@ +# Backend Crons + +Scheduled AWS Lambda jobs for Abstract Play: DynamoDB exports, static game records on S3, site-wide analytics, and live tournament/challenge automation. + +This repo complements [node-backend](/backend/) — the API writes live game state to DynamoDB; crons read that data (via daily exports or live queries) and publish derived artifacts to S3 and CloudFront. + +## Documentation + +- [Architecture](/crons/architecture/) — Serverless layout, schedules, layers, IAM +- [Getting started](/crons/getting-started/) — local setup, layers, invocation +- [Deployment](/crons/deployment/) — CI/CD, stages, upstream triggers +- [Records pipeline](/crons/pipeline/) — daily batch and summarize flow +- [Functions reference](/crons/functions/) — per-Lambda inputs, outputs, schedules +- [S3 outputs](/crons/s3-outputs/) — bucket keys and JSON shapes +- [Summarize](/crons/summarize/) — `_summary.json` metrics and rating logic +- [Live crons](/crons/live-crons/) — tournaments and standing challenges +- [Recommendation co-occurrence](/crons/recommendations-cooccur/) — `cooccur.json` for game recommendations +- [Recommendation analytics](/crons/recommendations-analytics/) — impression funnel rollups (private ops S3) + +## Key resources + +| Resource | Purpose | +|----------|---------| +| `abstractplay-db-dump` (S3) | DynamoDB point-in-time ION exports | +| `records.abstractplay.com` (S3 + CloudFront) | Published game records and analytics | +| `private-ops-153672715141-us-east-1-an` (S3) | Private ops artifacts (recommendation analytics) | +| `abstract-play-{stage}` (DynamoDB) | Live table (see [Database schema](/backend/database-schema/)) | + +EventBridge schedules run in **prod only** (`scheduleEnabled.prod: true` in [`serverless.yml`](../serverless.yml)). Dev stacks deploy the Lambdas but crons do not fire on a schedule. + +## Related docs + +- [Backend](/backend/) — API, DynamoDB schema, subsystems +- [Gameslib](/gameslib/) — `GameFactory`, `gameinfo`, rules engine used by record generation +- [Recranks](/recranks/) — `APGameRecord` format and rating engines + +*Last verified against `develop` branch.* diff --git a/crons/docs/live-crons.md b/crons/docs/live-crons.md new file mode 100644 index 00000000..37fd1e26 --- /dev/null +++ b/crons/docs/live-crons.md @@ -0,0 +1,106 @@ +# Live crons + +Functions that operate on **live DynamoDB** rather than only writing static artifacts. Most run on prod schedules and mutate data directly. + +## `dashboard-cruft-cleanup` + +**Schedule:** 03:00 UTC daily +**Source:** [`src/functions/dashboard-cruft-cleanup.ts`](../src/functions/dashboard-cruft-cleanup.ts) + +Prunes stale dashboard index cruft (`RECENTCOMPLETED#`, orphan `USERGAME#`) for users inactive ≥ 1 year. Candidates come from the daily S3 ION dump; each user is confirmed live before cleanup. Sets `USER.cleaned = true` (cleared on `me()` login in node-backend). + +See [Dashboard cruft cleanup](/crons/dashboard-cruft-cleanup/) for full detail. + +## `starttournaments` + +**Schedule:** 10:00 and 22:00 UTC daily +**Source:** [`src/functions/starttournaments.ts`](../src/functions/starttournaments.ts) + +### Purpose + +Automates the tournament lifecycle: + +1. Find tournaments that are ready to start (enough registered players, scheduled start time reached) +2. Cancel tournaments that cannot start (insufficient players) +3. Create initial games for started tournaments via `GameFactory` +4. Send notification emails through SES + +### DynamoDB access + +Queries and updates `TOURNAMENT` records on `abstract-play-{stage}`. Uses retry logic for throttling (`ThrottlingException`, etc.). + +### Gameslib integration + +- `gameinfo` — tournament-capable meta games and variant validation +- `GameFactory` — instantiate games from tournament configuration +- `GameBase` / `GameBaseSimultaneous` — create first moves for simultaneous-start games + +### Email / i18n + +Uses i18next with `apback` strings under `src/locales/` (exported from node-backend at build/CI time; see [Getting started](/crons/getting-started/#email-strings-apback)). Email templates reference tournament name, meta game, and player lists. + +### Related backend docs + +- [Tournaments subsystem](/backend/subsystems/tournaments/) +- [Database schema](/backend/database-schema/) — `TOURNAMENT` record shape + +### Manual invoke + +Scheduled runs sweep all eligible tournaments. For a **single tournament** (including resume after a partial start): + +```bash +serverless invoke -f starttournaments --stage prod --path invoke-resume.json +``` + +Example `invoke-resume.json`: + +```json +{"tournamentId":"","resume":true} +``` + +Resume requires `started: false`. Tournament start is **not** exposed via node-backend queries or the admin dashboard — only this Lambda. Ad-hoc prod ops scripts live under `bin/` (gitignored, local only), e.g. `bin/check-tournament-prod.mjs`, `bin/cleanup-tournament-prod.mjs`. + +## `inactive-challenge-cleanup` + +**Schedule:** 03:00 UTC daily +**Source:** [`src/functions/inactive-challenge-cleanup.ts`](../src/functions/inactive-challenge-cleanup.ts) + +Revokes open (`STANDINGCHALLENGE#`) and direct (`CHALLENGE`) challenges issued by players inactive ≥ 14 days (`USERS.lastSeen`). Pauses matching `REALSTANDING` presets and notifies acceptors (email, push, in-app for direct). + +See [Inactive challenge cleanup](/crons/inactive-challenge-cleanup/) for full detail. + +## `standingchallenges` + +**Schedule:** 00:00 and 12:00 UTC daily +**Source:** [`src/functions/standingchallenges.ts`](../src/functions/standingchallenges.ts) + +### Purpose + +Processes **preset standing challenge** requests stored as `REALSTANDING` records. When conditions are met (matching players online, preset rules satisfied), the cron creates challenge records in DynamoDB so the normal challenge flow in node-backend can pick them up. + +### DynamoDB access + +Queries `REALSTANDING` and related user/challenge records. Does not use gameslib — pure DynamoDB document operations. + +### Related backend docs + +- [Challenges subsystem](/backend/subsystems/challenges/) + +## Dev vs prod + +Both functions deploy to dev stacks but **EventBridge schedules are disabled on dev**. Test by invoking manually: + +```bash +serverless invoke -f dashboard-cruft-cleanup --stage prod +serverless invoke -f inactive-challenge-cleanup --stage prod +serverless invoke -f starttournaments --stage prod +serverless invoke -f standingchallenges --stage prod +``` + +Use prod with care — these mutate live data. + +## Related + +- [Functions reference](/crons/functions/) +- [Architecture](/crons/architecture/) +- [Deployment](/crons/deployment/) diff --git a/crons/docs/nav.json b/crons/docs/nav.json new file mode 100644 index 00000000..d10800ed --- /dev/null +++ b/crons/docs/nav.json @@ -0,0 +1,15 @@ +[ + { "slug": "index", "title": "Overview" }, + { "slug": "architecture" }, + { "slug": "getting-started", "title": "Getting started" }, + { "slug": "deployment" }, + { "slug": "pipeline", "title": "Records pipeline" }, + { "slug": "functions", "title": "Functions reference" }, + { "slug": "s3-outputs", "title": "S3 outputs" }, + { "slug": "recommendations-cooccur", "title": "Recommendation co-occurrence" }, + { "slug": "recommendations-analytics", "title": "Recommendation analytics" }, + { "slug": "summarize" }, + { "slug": "dashboard-cruft-cleanup", "title": "Dashboard cruft cleanup" }, + { "slug": "inactive-challenge-cleanup", "title": "Inactive challenge cleanup" }, + { "slug": "live-crons", "title": "Live crons" } +] diff --git a/crons/docs/pipeline.md b/crons/docs/pipeline.md new file mode 100644 index 00000000..343fea62 --- /dev/null +++ b/crons/docs/pipeline.md @@ -0,0 +1,113 @@ +# Records pipeline + +The batch pipeline runs **daily** (UTC) against the latest completed DynamoDB export in `abstractplay-db-dump`. Downstream jobs at 03:00 use whichever export finished most recently (by `manifest-summary.json` `LastModified`) — not necessarily the export started at that day's midnight `dumpdb` run. + +## Schedule overview + +All times UTC. Prod only. + +EventBridge cron expressions use `*` in the day-of-month field (daily). For example, `cron(0 0 * * ? *)` is **every day** at 00:00, not Sunday-only. Sunday-only would be `cron(0 0 ? * SUN *)`. + +| Time | Function(s) | Depends on | +|------|-------------|------------| +| Daily 00:00 | `dumpdb` | — | +| Daily 03:00 | `records`, `records-ttm`, `records-move-times`, `records-cooccur`, `records-rec-analytics`, `tournament-data` | Latest completed dump in `abstractplay-db-dump` (except `records-rec-analytics` — live DDB scan) | +| Daily 04:00 | `records-manifest` | Records batch outputs | +| Daily 06:00 | `summarize` | `ALL.json` in records bucket | +| Daily 06:15 | `player-summary-fanout` | Enqueues `player/*-summary.json` writes via SQS | +| Daily 07:30 | `records-manifest` | Post-summarize + player-slice refresh | + +Live crons (`starttournaments`, `standingchallenges`) run on separate daily schedules and query DynamoDB directly — see [Live crons](/crons/live-crons/). + +## Flow diagram + +```mermaid +flowchart TD + dumpdb["dumpdb daily 00:00 UTC"] --> batch["records + records-ttm + records-move-times + records-cooccur + tournament-data daily 03:00"] + recanalytics["records-rec-analytics daily 03:00"] --> ddb[(DynamoDB live)] + recanalytics --> s3ops[(private ops S3)] + batch --> manifest1["records-manifest daily 04:00"] + batch --> summarize["summarize daily 06:00"] + summarize --> fanout["player-summary-fanout daily 06:15"] + fanout --> sqs[(SQS)] + sqs --> workers["player-summary-worker"] + workers --> s3rec + fanout --> s3rec + fanout --> manifest2["records-manifest daily 07:30"] + live1["starttournaments 10:00 and 22:00 UTC"] --> ddb[(DynamoDB live)] + live2["standingchallenges 00:00 and 12:00 UTC"] --> ddb + dumpdb --> s3dump[(abstractplay-db-dump)] + batch --> s3rec[(records.abstractplay.com)] + summarize --> s3rec + manifest1 --> s3rec + manifest2 --> s3rec +``` + +## Step 1: Database export (`dumpdb`) + +Triggers a DynamoDB **point-in-time export** of the prod table (`abstract-play-prod`) to S3 bucket `abstractplay-db-dump` in ION format. AWS writes export files under `AWSDynamoDB/{uid}/data/*.ion.gz` plus a `manifest-summary.json`. + +The export is asynchronous — downstream jobs find the **latest** manifest by `LastModified` and process all data files for that export UID. Exports older than seven days are pruned by `dumpdb`. + +## Step 2: Parallel record generation (daily 03:00) + +Five Lambdas run in parallel at 03:00 UTC. Each reads the latest completed dump independently (they do not wait for each other's outputs). + +### `records` + +Reads GAME, TOURNAMENT, ORGEVENT, ORGEVENTGAME, and BOT records from the ION dump. For each completed game (`pk=GAME`, `sk` contains `#1#`), instantiates the rules engine via `GameFactory` and calls `genRecord()` to produce an [`APGameRecord`](/recranks/). + +Writes to `records.abstractplay.com`: + +- `ALL.json` — all game records +- `meta/{metaGame}.json` — per-game-type lists +- `player/{playerId}.json` — per-player lists +- `event/{eventId}.json` — tournament/event groupings + +### `records-ttm` + +Same dump ingestion, but computes per-player inter-move durations from the game stack. Writes `ttm/{playerId}.json` (array of milliseconds). + +### `records-move-times` + +Builds move-activity summaries over 7, 30, 180, and 365-day windows, plus site-wide move-time seasonality (DOW/hour bins) and **`weeklyActiveMovers`** (distinct players with ≥1 move per seven-day bucket). Writes `mvtimes.json`. + +### `records-cooccur` + +Scans completed GAME records and `USER` records (for `stars[]`) from the dump. Builds a PMI-normalized co-occurrence matrix for the front-end recommendation engine. Writes `recommendations/cooccur.json`. See [Recommendation co-occurrence](/crons/recommendations-cooccur/). + +### `records-rec-analytics` + +Scans live DynamoDB `RECOMMENDS#` impression events (not the ION dump). Computes anonymized funnel/CTR rollups and writes to the private ops bucket. See [Recommendation analytics](/crons/recommendations-analytics/). + +### `tournament-data` + +Extracts tournament records from the dump and writes `tournament-summary.json` plus `player/tournaments/{playerId}.json` per player. + +## Step 3: Manifest and CDN (`records-manifest`) + +Lists all objects in the records bucket, writes `_manifest.json` (v2 schema with `summaryFiles`). Does **not** invalidate CloudFront; see [S3 outputs — caching](/crons/s3-outputs/#cloudfront-and-caching). Runs at **04:00** and **07:30 UTC** so the late pass includes `_summary.json`, tier files, and `player/*-summary.json` from the summarize / fan-out pipeline. + +## Step 4: Summarize (daily 06:00) + +Reads `ALL.json`, computes site-wide analytics, writes `_summary.json` and tier files. See [Summarize](/crons/summarize/). + +## Step 5: Player summary fan-out (daily 06:15) + +`player-summary-fanout` loads `_summary-site.json`, `_summary-players.json`, and `_summary-ratings.json` in parallel (not the full monolith). It compares per-player content hashes against the previous `_summary-player-manifest.json` (v2) and enqueues SQS messages only for slices whose substantive content changed. When tier input is unchanged from the prior run, it skips all enqueues. + +`player-summary-worker` Lambdas (SQS-triggered, concurrency 25) write `player/{userId}-summary.json`. The handler returns structured metrics (`candidateCount`, `enqueuedCount`, `skippedCount`, `inputUnchanged`, `tierBytesLoaded`, `manifestBytes`) and logs a one-line summary. + +## Failure and timing + +- `dumpdb` starts at 00:00; the 03:00 batch uses the latest **completed** export, which may be from the previous day if today's export is still running +- If `records` fails, `ALL.json` is stale and `summarize` reflects old data +- `records-manifest` writes `_manifest.json` with current S3 listing — clients see whatever is at the origin after cache revalidation (no blanket invalidation) + +## Related + +- [Functions reference](/crons/functions/) +- [S3 outputs](/crons/s3-outputs/) +- [Recommendation co-occurrence](/crons/recommendations-cooccur/) +- [Recommendation analytics](/crons/recommendations-analytics/) +- [Summarize](/crons/summarize/) diff --git a/crons/docs/recommendations-analytics.md b/crons/docs/recommendations-analytics.md new file mode 100644 index 00000000..01bbb6c2 --- /dev/null +++ b/crons/docs/recommendations-analytics.md @@ -0,0 +1,99 @@ +# Recommendation impression analytics (`records-rec-analytics`) + +Nightly batch job that scans live DynamoDB `RECOMMENDS#` impression rows, computes anonymized funnel and CTR rollups, and writes reviewable artifacts to the **private ops S3 bucket**. Metrics only — no weight tuning, no front-end consumer. + +## Output bucket + +| Bucket | Prefix | +|--------|--------| +| `private-ops-153672715141-us-east-1-an` | `recommendations/analytics/` | + +**Not** published to `records.abstractplay.com` or CloudFront. + +## Schedule + +**Daily 03:00 UTC** — runs in parallel with `records-cooccur` and other 03:00 batch jobs. Unlike dump consumers, this Lambda **scans DynamoDB directly** (filtered `RECOMMENDS#` partition keys). + +## Inputs + +| Source | Filter | Fields | +|--------|--------|--------| +| DynamoDB `abstract-play-{stage}` | `pk` begins with `RECOMMENDS#`, `sk >= watermark` | `event`, `batchId`, `surface`, `tier`, plus event-specific attributes | + +Event types: `rec_show`, `rec_click`, `rec_challenge`. Schema matches [backend recommendations](/backend/subsystems/recommendations/). + +### Watermark + +`_state.json` stores `lastSkWatermarkMs` and processed `pk::sk` dedupe keys. Each run scans from `lastSkWatermarkMs - 5 minutes` (overlap buffer), skips keys already counted, then advances the watermark. + +First run (no state): looks back **7 days**. + +## Algorithm + +1. **Ingest** — parse DynamoDB items; strip `userId` from `pk` immediately (never written to output). +2. **Batch join** on `batchId` — funnel counts, orphan clicks, duplicate detection. +3. **Dimensional rollups** — `surface`, `tier`, `reasonType`, top meta-games (cap 30), position histogram (0–7). +4. **Rates** — CTR, challenge rate, end-to-end rate; dimensional CTR only when shows ≥ 20. +5. **Daily files** — merge new events into `daily/YYYY-MM-DD.json` (UTC from `sk` epoch). +6. **Rolling windows** — recompute `rolling7d` and `rolling30d` from retained daily files (~90 days). + +Implementation: [`src/utils/recAnalytics.ts`](../src/utils/recAnalytics.ts). Handler: [`src/functions/records-rec-analytics.ts`](../src/functions/records-rec-analytics.ts). + +## S3 layout + +| Key | Purpose | +|-----|---------| +| `recommendations/analytics/_state.json` | `lastRunAt`, `lastSkWatermarkMs`, `processedKeys` | +| `recommendations/analytics/daily/YYYY-MM-DD.json` | UTC day slice | +| `recommendations/analytics/summary.json` | Latest window + rolling 7d/30d | +| `recommendations/analytics/report/YYYY-MM-DD.md` | Human/agent-readable report | + +## `summary.json` schema (illustrative) + +```json +{ + "generatedAt": "2026-08-13T03:15:00.000Z", + "window": { "start": "2026-08-12T03:00:00.000Z", "end": "2026-08-13T03:15:00.000Z" }, + "totals": { "shows": 420, "clicks": 38, "challenges": 5 }, + "rates": { "ctr": 0.09, "challengeRate": 0.132, "endToEndRate": 0.012 }, + "bySurface": { "gamePicker": { "shows": 400, "clicks": 36, "ctr": 0.09 } }, + "byTier": { "warm": { "shows": 310, "clicks": 32, "ctr": 0.103 } }, + "byReasonType": { "content": { "clicks": 18, "showReasons": 200 } }, + "topClickedMetaGames": [{ "metaGame": "go", "count": 4 }], + "topChallengedMetaGames": [], + "positionHistogram": { "0": 12, "1": 8 }, + "rolling7d": { "totals": { "shows": 0, "clicks": 0, "challenges": 0 }, "rates": {} }, + "rolling30d": {}, + "dataQuality": { "eventsProcessed": 500, "parseErrors": 0, "orphanClicks": 2, "duplicateEventsPerBatch": 0 } +} +``` + +Published objects contain **only aggregates** — no user IDs, no raw `batchId` values. + +## Privacy + +- Raw events remain in DynamoDB with ~90-day TTL. +- Ops S3 artifacts are aggregate-only and private (IAM-restricted). + +## Manual invoke + +```bash +# dev +serverless invoke -f records-rec-analytics --stage dev + +# prod +serverless invoke -f records-rec-analytics --stage prod +``` + +Fetch results (requires AWS credentials with ops bucket access): + +```bash +aws s3 cp s3://private-ops-153672715141-us-east-1-an/recommendations/analytics/summary.json - +aws s3 cp s3://private-ops-153672715141-us-east-1-an/recommendations/analytics/report/2026-08-13.md - +``` + +## Related + +- [Recommendation co-occurrence](/crons/recommendations-cooccur/) — PMI matrix for the live recommender +- [Backend recommendations](/backend/subsystems/recommendations/) — event write path +- [Live crons](/crons/live-crons/) — other direct DynamoDB jobs diff --git a/crons/docs/recommendations-cooccur.md b/crons/docs/recommendations-cooccur.md new file mode 100644 index 00000000..80547124 --- /dev/null +++ b/crons/docs/recommendations-cooccur.md @@ -0,0 +1,88 @@ +# Recommendation co-occurrence (`records-cooccur`) + +Nightly batch artifact for the hybrid game recommender on the front end. Computes **PMI-normalized co-occurrence** between meta-games from player play history, with an optional **stars boost**. + +## Output + +| Key | Producer | +|-----|----------| +| `recommendations/cooccur.json` | `records-cooccur` | + +Public URL: `https://records.abstractplay.com/recommendations/cooccur.json` + +## Schedule + +**Daily 03:00 UTC** — runs in parallel with `records`, `records-move-times`, `records-ttm`, and `tournament-data`. Reads the latest completed DynamoDB ION dump (same pattern as other dump consumers). Does **not** depend on `player/*.json` from `records` (those are written in the same parallel window). + +Picked up by `records-manifest` at 04:00 UTC (or 07:30 after summarize and player-summary fan-out). + +## Inputs + +| Source (ION dump) | Field | Use | +|-------------------|-------|-----| +| Completed games `pk=GAME`, `sk` contains `#1#` | `metaGame`, `players[].id` | Per player: set of completed meta-games | +| `pk=USER` | `sk` (user id), `stars[]` | Optional boost: starred meta-games | + +### Stars boost + +For each player, the co-play set is: + +``` +coPlaySet = completedMetaGames ∪ starredMetaGames +``` + +Starred games count as co-played with each other and with completed games, even when the player has not finished a game in that meta-game. This matches the `user_names` / profile `stars` signal described in the recommendation design. + +Set `includeStarredBoost: true` in the artifact when stars were unioned in (always true for the current job). + +## Algorithm + +1. For each player with a non-empty `coPlaySet`, increment counts for every unordered pair `(A, B)` in the set. +2. `count(A)` = number of players whose `coPlaySet` contains `A`. +3. `N` = number of players with a non-empty `coPlaySet`. +4. PMI: + + ``` + PMI(A, B) = log( count(A,B) * N / (count(A) * count(B)) ) + ``` + +5. Keep pairs with `count(A,B) >= 5` (`DEFAULT_MIN_COOCCURRENCE`). +6. For each game `A`, store the top 20 neighbors by PMI descending. + +Implementation: [`src/utils/cooccurPmi.ts`](../src/utils/cooccurPmi.ts) (pure functions + unit tests). Handler: [`src/functions/records-cooccur.ts`](../src/functions/records-cooccur.ts). + +## JSON schema + +```json +{ + "generatedAt": "2026-08-13T00:00:00.000Z", + "minCooccurrence": 5, + "includeStarredBoost": true, + "games": { + "go": [ + { "metaGame": "amazons", "pmi": 1.42, "count": 87 }, + { "metaGame": "hex", "pmi": 1.18, "count": 54 } + ] + } +} +``` + +| Field | Meaning | +|-------|---------| +| `generatedAt` | ISO timestamp when the artifact was written | +| `minCooccurrence` | Minimum raw pair count threshold | +| `includeStarredBoost` | Whether `stars[]` were unioned into co-play sets | +| `games` | Map of meta-game → PMI neighbors (max 20 each) | + +## Front-end consumption + +The front-end `useGameRecommendations` hook fetches this artifact and passes it to `buildGameRecommendations` as `cooccurData`. Missing or failed fetch degrades to content + popularity only (`cooccurScore = 0`). + +Hybrid warm-tier weights (reference): 45% content, 35% co-occurrence, 15% popularity, 10% recency. + +## Related + +- [Records pipeline](/crons/pipeline/) +- [S3 outputs](/crons/s3-outputs/) +- [Backend recommendations subsystem](/backend/subsystems/recommendations/) — impression tracking (`RECOMMENDS#`) +- [Recommendation analytics](/crons/recommendations-analytics/) — nightly funnel rollups (ops S3) diff --git a/crons/docs/s3-outputs.md b/crons/docs/s3-outputs.md new file mode 100644 index 00000000..e80ab62a --- /dev/null +++ b/crons/docs/s3-outputs.md @@ -0,0 +1,236 @@ +# S3 outputs + +Static artifacts are published to **`records.abstractplay.com`** (S3 + CloudFront). DynamoDB exports land in **`abstractplay-db-dump`** daily (`dumpdb`); batch jobs read the latest completed export. + +## Records bucket layout + +| Key pattern | Producer | Description | +|-------------|----------|-------------| +| `ALL.json` | `records` | Array of all [`APGameRecord`](/recranks/) objects | +| `_summary.json` | `summarize` | Full site-wide analytics monolith — see [Summarize](/crons/summarize/) | +| `_summary-site.json` | `summarize` | Tier 0 site overview (lazy-load bootstrap) | +| `_summary-players.json` | `summarize` | Tier 1 per-player bulk stats | +| `_summary-ratings.json` | `summarize` | Tier 2 ratings bulk (Glicko-enriched) | +| `player/{playerId}-summary.json` | `player-summary-worker` | Per-player summary slice (~few KB) | +| `_summary-player-manifest.json` | `player-summary-fanout` | Fan-out manifest v2: `candidateCount`, `expectedCount` (enqueued this run), `skippedCount`, `inputFingerprint`, `contentHashes` | +| `_manifest.json` | `records-manifest` | S3 object listing + `summaryFiles` (v2) | +| `meta/{metaGame}.json` | `records` | Game records filtered by meta game name | +| `player/{playerId}.json` | `records` | Game records for one player | +| `event/{eventId}.json` | `records` | Game records for a tournament or org event | +| `ttm/{playerId}.json` | `records-ttm` | Array of inter-move durations (milliseconds) | +| `mvtimes.json` | `records-move-times` | Move activity counts by meta game and time window | +| `recommendations/cooccur.json` | `records-cooccur` | PMI co-occurrence matrix for game recommendations | +| `tournament-summary.json` | `tournament-data` | Per-player tournament aggregate stats | +| `player/tournaments/{playerId}.json` | `tournament-data` | Individual tournament results for one player | + +## Game record format + +Each record in `ALL.json`, `meta/`, `player/`, and `event/` files conforms to the **APGameRecord** schema documented in [Recranks](/recranks/). Records are produced by calling `GameFactory(metaGame, state).genRecord(...)` in [`records.ts`](../src/functions/records.ts). + +Key header fields used downstream: + +- `header.site.gameid` — stable composite id (see below); summarize parses meta UID and variant codes from this field +- `header.game.name` — meta game display name (human-readable; not used as summarize map keys) +- `header.game.variants` — localized variant labels at record-generation time +- `header.players[].userid` — player ID +- `header["date-start"]`, `header["date-end"]` — ISO timestamps +- `moves` — move history (timeout/abandoned detection in summarize) + +### Game record `gameid` + +`header.site.gameid` is set in [`records.ts`](../src/functions/records.ts) via `encodeRecordGameId()` ([`recordGameId.ts`](../src/utils/recordGameId.ts)): + +| Format | Example | Notes | +|--------|---------|-------| +| Current | `{uuid}#{metaGame}:{sortedVariantUids}` | Variant UIDs joined with `\|`; trailing colon when no variants (`…#chess:`) | +| Legacy | `{metaGame}#{uuid}` | Pre-encoding records; summarize treats variant codes as unknown | + +Parse/decode helpers: `parseRecordGameId`, `encodeRecordGameId`, `variantComboKey` in [`src/utils/recordGameId.ts`](../src/utils/recordGameId.ts). + +### Summary `game` keys + +`_summary*.json` map keys and `ratings.highest[].game` strings use **meta game UIDs**, not display names: + +| Key shape | Example | +|-----------|---------| +| Meta only | `go` | +| Meta + variants | `go (size-9)` | +| Implicit defaults (variant groups) | `akimbo (#board\|#ruleset)` — empty `gameid` variant segment uses `#group` sentinels via gameslib `variantUidsForBatchRating` | +| Explicit no variants | `chess (no variants)` — only when the meta game has **no** `gameinfo.variants` dimension | + +Front-end clients resolve UIDs to localized display names via gameslib (`src/lib/summaryGameKeys.js` in the front repo). + +## `_summary.json` and tier files + +The **monolith** (`_summary.json`) is typed as `StatSummary` in [`src/types/stats/StatSummary.ts`](../src/types/stats/StatSummary.ts). Tier types: [`StatSummaryTiers.ts`](../src/types/stats/StatSummaryTiers.ts). + +| Key | Type | Role | +|-----|------|------| +| `_summary.json` | `StatSummary` | Full superset; backward compatible download / batch consumers | +| `_summary-site.json` | `StatSummarySite` | Site stats, geo, histograms (site keys), `metaStats`, `plays`, `topPlayers` | +| `_summary-players.json` | `StatSummaryPlayers` | `players.*` + `histograms.players` / `playerTimeouts` + `pastDisplayNames` (per-user alias list from record headers) | +| `_summary-ratings.json` | `StatSummaryRatings` | `ratings.*` including Glicko aggregates | +| `player/{userId}-summary.json` | `PlayerSummarySlice` | One user's Tier 1/2 subset; optional `pastDisplayNames` (distinct embedded record names excluding current `USERS.name`) | + +All JSON objects use `Content-Type: application/json`. Each tier/slice includes `generated` (ISO timestamp). + +### Monolith top-level fields + +| Field | Meaning | +|-------|---------| +| `numGames`, `numPlayers` | Totals from `ALL.json` | +| `oldestRec`, `newestRec` | Date range of completed games | +| `timeoutRate` | Fraction of games with clock timeout **or** abandonment | +| `abandonedRate` | Fraction of games closed by abandonment only | +| `playContext` | Casual vs tournament/org-event game counts | +| `pieRates` | Pie invocation rates for supported meta games | +| `playerCountMix` | Player-count distribution for multi-player metas | +| `ratings` | ELO/Glicko/Trueskill aggregates (`highest`, `avg`, `weighted`, `glickoByGame`, `glickoSite`, `glickoMeta`) | +| `topPlayers` | Top-rated player/game pairs | +| `plays`, `players` | Game and player activity rankings | +| `histograms` | Play-count distributions, first-timers, returning players, weekly active movers, timeout/abandonment rates | +| `metaStats` | Per-game two-player stats (length, first-player win rate, draw rate) | +| `geoStats` | Registered user counts by country (from live USERS table) | +| `activeGeoStats` | Players who completed a game in the past 30 days, by profile country | +| `rivalries` | Two-player pair frequencies (≥50 shared games); anonymized unless both players opted in (`players` array when named) | +| `seasonality` | Move-time activity by UTC day/hour (from `mvtimes.json`; last 365 days) | +| `hoursPer` | `{ mean, median, n, byWeek }` — winsorized (p2–p98) hours per move site-wide | + +Full field documentation: [Summarize](/crons/summarize/). + +### `_summary-player-manifest.json` (fan-out manifest v2) + +Written by `player-summary-fanout` after each run: + +| Field | Meaning | +|-------|---------| +| `version` | `2` | +| `generated` | Same `generated` timestamp as tier files | +| `enqueuedAt` | ISO timestamp when fan-out finished | +| `candidateCount` | All-time players considered for slices | +| `expectedCount` | SQS messages enqueued this run (changed slices only) | +| `skippedCount` | Candidates skipped because slice content hash matched prior run | +| `inputFingerprint` | SHA-256 of substantive tier content (excludes `generated` / `tier`) | +| `contentHashes` | Per-user slice content hashes for the next run's skip logic | + +First deploy after this change enqueues all candidates (no v2 manifest yet). Legacy v1 manifests are treated as having no stored hashes. + +## `mvtimes.json` + +Produced by `records-move-times`. Object with: + +| Key | Meaning | +|-----|---------| +| `raw1w`, `raw1m`, `raw6m`, `raw1y` | Move counts by meta game in each rolling window | +| `players1w`, … | Distinct active players by meta game per window | +| `playersSum1w`, … | Cumulative unique-player scores by meta game | +| `seasonality` | Site-wide move activity bins (last 365 days): `{ movesByDow, playersByDow, movesByHour, windowDays }` | +| `weeklyActiveMovers` | `{ originMs, byWeek }` — distinct players with ≥1 move per seven-day bucket (same origin as completion histograms; move data ~1y) | + +`summarize` copies `seasonality` and aligns `weeklyActiveMovers` into `_summary.json` (`histograms.activeMovers`). + +## `recommendations/cooccur.json` + +PMI-normalized co-occurrence neighbors per meta-game for the hybrid recommender. Includes optional stars boost (`includeStarredBoost`). Full schema and algorithm: [Recommendation co-occurrence](/crons/recommendations-cooccur/). + +## Private ops bucket (`private-ops-153672715141-us-east-1-an`) + +Not served via CloudFront. IAM-restricted. + +| Key pattern | Producer | Description | +|-------------|----------|-------------| +| `recommendations/analytics/_state.json` | `records-rec-analytics` | Watermark and dedupe state | +| `recommendations/analytics/daily/YYYY-MM-DD.json` | `records-rec-analytics` | UTC daily impression rollups | +| `recommendations/analytics/summary.json` | `records-rec-analytics` | Latest window + rolling 7d/30d | +| `recommendations/analytics/report/YYYY-MM-DD.md` | `records-rec-analytics` | Human/agent-readable report | +| `stats/rivalries.json` | `summarize` | All qualifying rivalry pairs with user IDs and display names (not anonymized; min 5 shared games) | + +**Retired (preview-era layout feedback):** `gamemove-layout/analytics/*` on private ops S3 is an archived snapshot from the removed `layout-feedback-analytics` cron. No new writes after teardown; DynamoDB `LAYOUTFB#*` rows are purged via `node-backend` `bin/purge-layout-feedback-events.mjs`. + +See [Recommendation analytics](/crons/recommendations-analytics/) and [Summarize](/crons/summarize/). + +## Dump bucket layout + +Exports from `dumpdb` appear under: + +``` +AWSDynamoDB/{export-uid}/manifest-summary.json +AWSDynamoDB/{export-uid}/data/*.ion.gz +``` + +Batch functions locate the newest `manifest-summary.json`, extract the UID, and read all `data/*.gz` files for that export. + +## `_manifest.json` (v2) + +Typed in [`src/utils/recordsManifest.ts`](../src/utils/recordsManifest.ts). The records bucket manifest is no longer a bare S3 `Contents` array. + +```typescript +{ + version: 2, + generated: string, // ISO timestamp when manifest was built + summaryFiles: { + monolith: { key, lastModified?, size? }, + site: { key, lastModified?, size? }, + players: { key, lastModified?, size? }, + ratings: { key, lastModified?, size? }, + playerSummaryPattern: "player/{userId}-summary.json", + playerManifest: { key: "_summary-player-manifest.json", ... } + }, + objects: _Object[] // full bucket listing (same data as legacy root array) +} +``` + +**Backward compatibility:** legacy consumers expecting a root array should use `Array.isArray(data) ? data : data.objects`. + +`records-manifest` runs at **04:00** and **07:30 UTC** (late pass is after `summarize` at 06:00 and `player-summary-fanout` at 06:15). The handler logs a warning if any required `_summary*.json` tier key is missing from the listing. + +## CloudFront and caching + +**Distribution:** `EM4FVU08T5188` — `https://records.abstractplay.com` + +CloudFront **does not** run blanket `/*` invalidations (removed to avoid quota/cost issues). Freshness relies on S3 object headers set by crons. + +### S3 cache headers (all records-bucket JSON) + +| Object type | `Cache-Control` | `Content-Type` | +|-------------|-----------------|----------------| +| Daily batch JSON (`ALL.json`, `meta/*`, `player/*`, `_summary*.json`, etc.) | `public, max-age=0, must-revalidate` | `application/json` | +| `_manifest.json` | `no-cache` | `application/json` | + +After each daily cron overwrite, the next CDN/browser request revalidates with S3 (`If-None-Match`); changed objects return a new body without invalidation. + +Implemented in [`src/utils/recordsJson.ts`](../src/utils/recordsJson.ts) (`putRecordsJson`). + +### Gzip compression + +CloudFront compresses responses only when the **origin** returns a compressible `Content-Type` (e.g. `application/json`). Objects previously uploaded as `application/octet-stream` were not compressed. + +1. **Crons** — set `Content-Type: application/json` on upload (above). +2. **CloudFront** — enable **Compress objects automatically** on the default cache behavior. + +One-time enable (prod credentials): + +```bash +npm run enable-records-cdn-compress +# preview: npm run enable-records-cdn-compress -- --dry-run +``` + +Or in the AWS Console: CloudFront → distribution `EM4FVU08T5188` → **Behaviors** → Edit default → **Compress objects automatically: Yes**. + +Verify after deploy + CF propagation: + +```bash +curl.exe -sI -H "Accept-Encoding: gzip" https://records.abstractplay.com/_summary-site.json +``` + +Expect `Content-Encoding: gzip` and `Content-Length` much smaller than the uncompressed object. + +Clients should use `_manifest.json` `summaryFiles` or tier URLs rather than hard-coding cache assumptions. + +## Related + +- [Records pipeline](/crons/pipeline/) +- [Recommendation co-occurrence](/crons/recommendations-cooccur/) +- [Recommendation analytics](/crons/recommendations-analytics/) +- [Recranks schema reference](/recranks/schema-reference/) +- [Functions reference](/crons/functions/) diff --git a/crons/docs/summarize.md b/crons/docs/summarize.md new file mode 100644 index 00000000..e4547994 --- /dev/null +++ b/crons/docs/summarize.md @@ -0,0 +1,310 @@ +# Summarize + +The `summarize` Lambda reads `ALL.json` from the records bucket, computes site-wide statistics, and writes summary artifacts to the records bucket. It also writes full rivalry pair data (with user IDs) to the private ops bucket. It runs **daily at 06:00 UTC** so analytics stay current even mid-week as new games complete. + +Source: [`src/functions/summarize.ts`](../src/functions/summarize.ts). Pure helpers and unit tests: [`src/functions/summarizeHelpers.ts`](../src/functions/summarizeHelpers.ts), [`summarizeHelpers.test.ts`](../src/functions/summarizeHelpers.test.ts), [`summarizeSolo.ts`](../src/functions/summarizeSolo.ts), [`summarizeSolo.test.ts`](../src/functions/summarizeSolo.test.ts). + +## Input + +1. **`ALL.json`** — full array of [`APGameRecord`](/recranks/) from the `records` cron +2. **`mvtimes.json`** — move-time seasonality (produced by `records-move-times` at 03:00 UTC) +3. **Live DynamoDB** — `USERS` partition query for country codes (geo stats) + +## Output + +| Destination | Key | Content | +|-------------|-----|---------| +| Records bucket | `_summary.json` | Full `StatSummary` monolith (backward compatible) | +| Records bucket | `_summary-site.json` | Tier 0 — site overview (`StatSummarySite`) | +| Records bucket | `_summary-players.json` | Tier 1 — per-player bulk (`StatSummaryPlayers`) | +| Records bucket | `_summary-ratings.json` | Tier 2 — ratings bulk (`StatSummaryRatings`) | +| Records bucket | `player/{userId}-summary.json` | Per-player slice for profile / quick-picks | +| Records bucket | `_summary-player-manifest.json` | Fan-out manifest v2 (`candidateCount`, `expectedCount` = enqueued, `skippedCount`, `inputFingerprint`, `contentHashes`) | +| Private ops bucket | `stats/rivalries.json` | Full rivalry pairs with user IDs | + +Per-player slices are written by **`player-summary-fanout`** (daily **06:15 UTC**), which enqueues one SQS message per **changed** user slice; **`player-summary-worker`** performs the S3 puts. Unchanged slices are skipped via content hashes stored in `_summary-player-manifest.json` v2. + +Typed as `StatSummary` in [`src/types/stats/StatSummary.ts`](../src/types/stats/StatSummary.ts). See also [S3 outputs](/crons/s3-outputs/). + +## Top-level fields (`StatSummary`) + +| Field | Meaning | +|-------|---------| +| `numGames`, `numPlayers` | Totals from completed games in `ALL.json` | +| `oldestRec`, `newestRec` | ISO date range of completed games | +| `timeoutRate` | Fraction of games ending by **clock timeout or abandonment** (site-wide) | +| `abandonedRate` | Fraction ending by **abandonment only** (stale soft-clock games) | +| `playContext` | `{ casual, event }` — games without vs with tournament/org event linkage | +| `pieRates` | Per meta game: `{ game, n, pied, rate }` where pie is supported | +| `playerCountMix` | Per meta game supporting 3+ players: `{ game, byCount: { "3": n, … } }` | +| `ratings` | ELO / Glicko-2 / TrueSkill aggregates (`highest`, `avg`, `weighted`, `glickoByGame`, `glickoSite`, `glickoMeta`) | +| `topPlayers` | Top-rated player/game pairs | +| `plays`, `players` | Game and player activity rankings | +| `histograms` | Weekly play distributions (see below) | +| `recent` | Meta games with the most recent completions | +| `hoursPer` | Structured pacing stats (see below) | +| `metaStats` | Per-meta two-player stats including `drawRate` | +| `soloMetaStats` | Per meta + variant solo aggregates (`attempts`, `uniquePlayers`, score medians, outcome breakdowns) | +| `soloSeedBoards` | Per meta + variant + `challenge-seed` leaderboards (best attempt per user, with `attempts` count) | +| `hMeta` | Per-meta h-index (breadth of participation) | +| `geoStats` | Registered users by country (live `USERS` table) | +| `activeGeoStats` | Players who completed a game in the past 30 days, by profile country | +| `rivalries` | Anonymized two-player pair frequencies (public pairs with ≥50 shared games) | +| `seasonality` | Move-time activity by UTC day/hour (copied from `mvtimes.json`; last 365 days) | + +## Processing stages + +### Segmentation + +Each record is indexed by: + +- Meta game UID parsed from `header.site.gameid` (see [S3 outputs — gameid](/crons/s3-outputs/#game-record-gameid)); legacy records fall back to display-name lookup via gameslib +- Player user IDs +- Date range (`oldestRec`, `newestRec`) +- Timeout vs abandoned moves (via `summarizeHelpers`) +- Play context (casual vs event) +- Pie invocation and multi-player counts where supported + +### Meta statistics (`metaStats`) + +For each meta game UID (and variant subgroup when multiple variant combinations exist), computes two-player stats. Map keys match `batchRatingGameLabel()` output after variant canonicalization (same rules as ratings). + +- Game count (`n`) +- Average and median move count (`lenAvg`, `lenMedian`) +- First-player win rate (`winsFirst`) +- Draw rate (`drawRate`) + +Only games with exactly two players and more than two moves are included. + +### Solo statistics (`soloMetaStats`, `soloSeedBoards`) + +1-player records (`header.players.length === 1`) are aggregated separately from two-player `metaStats` and ratings. Each archived solo run is its own row in `ALL.json` — retries on the same `challenge-seed` are kept and grouped at summarize time. + +**`soloMetaStats`** — keyed like ratings (`batchRatingGameLabel`, e.g. `puzzle (standard)`): + +| Field | Meaning | +|-------|---------| +| `attempts` | Total archived solo runs (includes retries) | +| `uniquePlayers` | Distinct `userid` values | +| `repeatAttemptRate` | `(attempts - uniquePlayers) / attempts` | +| `outcomeTypes` | Counts by `header.outcome-type` (`binary`, `graded`, `score`, `timed`) | +| `scoreMedianAllAttempts` | Median of all attempt scores | +| `scoreMedianBestPerUser` | Median of each user's best score on that variant | +| `passRateAllAttempts` / `passRateBestPerUser` | Binary outcomes only | +| `gradeHistogramBestPerUser` | Grade counts from each user's best attempt | +| `moveCountMedian` | Median round count across attempts | + +**`soloSeedBoards`** — one entry per meta + variant + `challenge-seed` (seeded runs only): + +| Field | Meaning | +|-------|---------| +| `rows` | Leaderboard sorted by `score-direction`; one row per user (best attempt) | +| `rows[].attempts` | How many times that user played this seed | +| `attempts`, `uniquePlayers` | Pool totals for the seed | +| `scoreMedianAllAttempts`, `scoreMedianBestPerUser` | Same split as meta stats | + +Tie-break for best attempt: better score (per direction), then fewer moves, then earlier `date-end`. + +### Timeout vs abandoned + +| Metric | Scope | Includes | +|--------|-------|----------| +| `timeoutRate` | Site-wide | Clock timeouts **and** abandonments | +| `abandonedRate` | Site-wide | Abandonments only | +| `histograms.timeouts` | Weekly rates | Clock timeouts only | +| `histograms.abandoned` | Weekly rates | Abandonments only | +| `players.timeouts` | Per player | **Removed** — use `players.timeoutStats` (`count`, `latestTimeoutMs`) | +| `players.timeoutStats` | Per timed-out player | `{ user, count, latestTimeoutMs }` — one row per user with ≥1 clock timeout | + +Detection uses `recordHasTimeout` / `recordHasAbandoned` and `findTimeoutPlayerSeat` in [`summarizeHelpers.ts`](../src/functions/summarizeHelpers.ts). + +### H-index metrics + +- **`hMeta`** — per-meta-game h-index (breadth of player participation) +- **`players.h`, `players.hOpp`** — player h-index and opponent h-index site-wide + +Uses `gameinfo` from gameslib to map display names to meta UIDs. + +### Ratings (`ratings`) + +Game keys in `ratings.highest[].game`, `glickoByGame[].game`, and `topPlayers[].game` are **meta UIDs** plus optional variant UIDs, produced by `batchRatingGameLabel()` in [`batchRatings.ts`](../src/lib/batchRatings.ts) after [`variantUidsForBatchRating`](https://github.com/AbstractPlay/gameslib) canonicalizes record UIDs (e.g. `go (size-9)`, `akimbo (#board|#ruleset)`, `chess (no variants)`). An empty variant list on a rated game with variant **groups** is not `(no variants)`; it resolves to implicit `#group` defaults (and merges with explicit UIDs that are rating-equivalent to `[]`, such as Akimbo `size-13`). Tournament seeding passes `tournament.metaGame` and `tournament.variants` through the same canonicalization before lookup. + +For each meta game UID (and variant subgroup), runs three rating engines from `@abstractplay/recranks`: + +| Engine | Class | Notes | +|--------|-------|-------| +| ELO | `ELOBasic` | Default batch rating | +| Glicko-2 | `Glicko2` | Period-based; 60-day periods via `GLICKO_PERIOD_MS`; prior **1200 / 350** (aligned with batch Elo start) | +| TrueSkill | `Trueskill` | `betaStart: 25/9` | + +Outputs: + +- `ratings.highest` — per user/game rows with Elo (`rating`), W/L/D, full `glicko` (`GlickoStats`), and `trueskill`. **Legacy name** — rows are all rated players, not “highest only”; the enriched **`glicko` object is canonical** for Glicko consumers (use `ratingLow`, `provisional`, etc.). +- `ratings.avg` — simple Elo averages across metas per user +- `ratings.weighted` — Elo weighted by games played per user +- `ratings.glickoByGame` — flat Glicko-only rows: `{ user, game, glicko }` (same pool as `highest`) +- `ratings.glickoSite` — per-user cross-meta composite: weighted `rating`, `rd`, `ratingLow` / `ratingHigh`, `n`, plus `provisional` / `established` (true if any game row matches) +- `ratings.glickoMeta` — thresholds, `periodMs`, `generatedAt`, and run counts (`counts.byGame`, `counts.site`) +- `ratings.playerCountsByUid` — distinct rated players per meta game UID (across all variant rows); replaces DynamoDB `ratingsCount` for display once consumers switch + +**Primary rank metric:** conservative Glicko `ratingLow` (`rating − 2×rd`). `topPlayers` and tournament seeding use this ordering; batch Elo `rating` remains a secondary column. + +#### Glicko row shape (`GlickoStats`) + +Each `glicko` object on `ratings.highest` and `ratings.glickoByGame`: + +| Field | Meaning | +|-------|---------| +| `rating`, `rd`, `volatility` | Full Glicko-2 state (μ, φ, σ) | +| `ratingLow`, `ratingHigh` | `rating ± 2×rd` (95% interval; use `ratingLow` for conservative seeding/sort) | +| `provisional` | `n < 10` **or** `rd > 200` | +| `established` | `n >= 20` **and** `rd <= 110` | +| `n` | Rated games in that meta/variant pool | + +`glickoMeta` repeats the threshold constants (`establishedRd`, `provisionalRd`, `minGamesEstablished`, `minGamesProvisional`) so consumers can apply stricter rules without redeploying crons. + +### Tiered exports + +The monolith remains the full contract. Three tier files are **views** for lazy front-end loading (see [S3 outputs](/crons/s3-outputs/)): + +| Tier file | `tier` | Contents | +|-----------|--------|----------| +| `_summary-site.json` | `site` | Site aggregates, geo, seasonality, rivalries, `histograms` site keys, `metaStats`, `plays`, `topPlayers` | +| `_summary-players.json` | `players` | `players.*`, `histograms.players`, `histograms.playerTimeouts` | +| `_summary-ratings.json` | `ratings` | Full `ratings` object | + +Each tier/slice includes `generated` (ISO timestamp). Uploaded with `Content-Type: application/json` and `Cache-Control: public, max-age=0, must-revalidate` (see [S3 outputs — CloudFront and caching](/crons/s3-outputs/#cloudfront-and-caching)). + +### Player rankings (`players`, `topPlayers`) + +- **`topPlayers`** — highest `ratingLow` per rated game/variant key; full `UserGameRating` row (`glicko`, Elo `rating`, `wld`) + +- **`social`** — players with the most distinct opponents +- **`eclectic`** — players who played the widest variety of meta games +- **`allPlays`** — total games played +- **`timeoutStats`** — per-user clock-timeout aggregates (`count`, `latestTimeoutMs`; not abandonments). Weekly charts use `histograms.playerTimeouts`. + +### Histograms (`histograms`) + +| Key | Meaning | +|-----|---------| +| `all` | Completed games per week | +| `allPlayers` | Distinct players **completing** a game per week | +| `activeMovers` | Distinct players who made **≥1 move** per week (from move timestamps; ~1y of move data) | +| `meta`, `players` | Per-game and per-player weekly counts | +| `playerTimeouts` | Per-player timeout counts over time | +| `firstTimers` | Users completing their first game that week | +| `returningPlayers` | Users who played that week but first played in an earlier week | +| `timeouts` | Clock-timeout rate per week | +| `abandoned` | Abandonment rate per week | + +Week buckets align from `oldestRec`; the right-most bucket may be partial. + +**`activeMovers` vs `allPlayers`:** finishers vs players who moved in any game that week (including in-progress async games). Sourced from `mvtimes.json`; only ~365 days of move timestamps are available, so early buckets may be zero. + +### Recent activity (`recent`) + +Games with the most recent `date-end` values. + +### Hours per move (`hoursPer`) + +Structured object (`HoursPerStats`): + +| Field | Meaning | +|-------|---------| +| `mean` | Move-weighted mean of winsorized per-game rates | +| `median` | Median of winsorized per-game rates | +| `n` | Number of qualifying games | +| `byWeek` | Median winsorized hours per move per week bucket | + +Excludes games ending by clock timeout or abandonment and games with fewer than two move rounds. Per-game rates are **winsorized at the 2nd and 98th percentiles** so extreme outliers (very slow correspondence games, bad timestamps) do not dominate the summary. + +### Geographic stats + +- **`geoStats`** — all registered users with a recognized country on their profile (`pk=USERS` query, `isoToCountryCode`) +- **`activeGeoStats`** — players who completed at least one game in the **past 30 days** (from `ALL.json` completion timestamps), grouped by the same country mapping + +### Play context (`playContext`) + +Counts games with no event/tournament linkage (`casual`) vs those tied to an org or tournament event (`event`). + +### Pie rates (`pieRates`) + +For meta games whose gameslib flags include `pie` or `pie-even`: total games, count where pie was invoked (`header.pied` or `header["pie-invoked"]`), and rate. + +### Player count mix (`playerCountMix`) + +For meta games that support more than two players: histogram of completed games by player count. + +### Rivalries + +Two-player completed games only. Pairs are canonicalized (`userA < userB`). Only pairs with at least **5** shared games (`RIVALRY_MIN_GAMES`) are included. + +**Private ops** (`stats/rivalries.json`): every qualifying pair with `{ userA, nameA, userB, nameB, n }` — user IDs and display names (not anonymized). + +**Public** (`_summary.json` → `rivalries`): every pair with at least **50** shared games (`RIVALRY_PUBLIC_MIN_GAMES`), ordered by `n`. Pairs are anonymized as `{ rank, label: "Pair N", n }` unless **both** players have `publicRivalries: true` on their `USERS` record. Opted-in pairs include `players: [{ id, name }, { id, name }]` and `label` as `"NameA vs NameB"`. + +```json +{ "rank": 1, "label": "Pair 1", "n": 42 } +``` + +```json +{ + "rank": 1, + "label": "Alice vs Bob", + "n": 42, + "players": [ + { "id": "…", "name": "Alice" }, + { "id": "…", "name": "Bob" } + ] +} +``` + +```json +{ + "generated": "2026-08-14T12:00:00.000Z", + "minGames": 5, + "pairs": [{ "userA": "…", "nameA": "Alice", "userB": "…", "nameB": "Bob", "n": 42 }] +} +``` + +### Seasonality (`seasonality`) + +**Not computed in this Lambda.** Copied from `mvtimes.json` → `seasonality`, which is built by [`records-move-times`](../src/functions/records-move-times.ts) from per-move `_timestamp` values on the game stack (last **365** days). This reflects when players actually move, not when async games finish. + +| Field | Length | Meaning | +|-------|--------|---------| +| `movesByDow` | 7 | Move count by UTC day-of-week (`0` = Sunday … `6` = Saturday), aggregated across the window | +| `playersByDow` | 7 | Distinct players who made at least one move on that weekday (across the window) | +| `movesByHour` | 24 | Move count by UTC hour (`0`–`23`) | +| `windowDays` | — | Rolling window length (365) | + +See [`moveSeasonality.ts`](../src/utils/moveSeasonality.ts). + +## Helper module + +[`summarizeHelpers.ts`](../src/functions/summarizeHelpers.ts) exports: + +| Function / constant | Purpose | +|---------------------|---------| +| `recordHasTimeout` / `recordHasAbandoned` | Detect end-of-game timeout states | +| `findTimeoutPlayerSeat` | Identify which player timed out (clock only) | +| `computeTimeoutHistogramRates` | Weekly clock-timeout rates | +| `computeHoursPerStats` | Winsorized mean, median, and weekly trend for hours per move | +| `computeReturningPlayersPerWeek` | Returning-player histogram | +| `recordWasPied` / `gameSupportsPie` | Pie rule stats | +| `gameSupportsMultiPlayerCount` | Multi-player-capable metas | +| `computeRivalryPairs` / `anonymizeRivalries` / `publishRivalries` / `enrichRivalryPairsWithDisplayNames` | Rivalry aggregation | +| `partitionByGlickoPeriod` / `computeGlickoNumPeriods` | Glicko rating periods | +| `GLICKO_PERIOD_MS` | 60-day period constant | +| `RIVALRY_MIN_GAMES`, `RIVALRY_PUBLIC_MIN_GAMES` | Rivalry thresholds (ops vs public) | + +## Custom JSON serialization + +Uses `replacer` from gameslib serialization when stringifying nested maps in rating output. + +## Related + +- [S3 outputs](/crons/s3-outputs/) +- [Recranks](/recranks/) — rating engines and record schema +- [Records pipeline](/crons/pipeline/) diff --git a/crons/package.json b/crons/package.json new file mode 100644 index 00000000..412397ad --- /dev/null +++ b/crons/package.json @@ -0,0 +1,88 @@ +{ + "name": "abstractplay-backend-crons", + "version": "1.0.0-beta", + "description": "node.js lambdas of the periodic cron jobs", + "type": "module", + "scripts": { + "clean": "rimraf dist .esbuild .serverless", + "pretest": "node scripts/ensure-apback-locales.mjs", + "prebuild:layers": "node scripts/ensure-apback-locales.mjs", + "test": "vitest run", + "test:watch": "vitest", + "build-ts": "tsc", + "typecheck": "tsc --noEmit", + "lint": "eslint .", + "build": "npm run lint", + "build:layers": "node scripts/build-layers.mjs", + "test:layers": "npm run build:layers && node scripts/smoke-layer-modules.mjs", + "sync-apback-locales": "node scripts/sync-apback-locales.mjs", + "check-apback-locales": "node scripts/check-apback-locales.mjs", + "preview-inactive-challenges": "tsx scripts/preview-inactive-challenge-cleanup.ts", + "run-records-pipeline": "node scripts/run-records-pipeline.mjs", + "enable-records-cdn-compress": "node scripts/enable-records-cdn-compress.mjs", + "deploy-dev": "serverless deploy", + "deploy-prod": "serverless --stage prod deploy", + "full-dev": "npm run clean && npm run build && serverless deploy", + "full-prod": "npm run build && serverless --stage prod deploy", + "sync-deps": "node ../scripts/sync-crons-ap-deps.mjs --stage dev", + "sync-deps:prod": "node ../scripts/sync-crons-ap-deps.mjs --stage prod" + }, + "author": "Paul van Wamelen", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/AbstractPlay/node-backend.git", + "directory": "crons" + }, + "bugs": { + "url": "https://github.com/AbstractPlay/node-backend/issues" + }, + "homepage": "https://github.com/AbstractPlay/node-backend/tree/develop/crons#readme", + "engines": { + "node": ">=24", + "npm": "11.6.2" + }, + "packageManager": "npm@11.6.2", + "dependencies": { + "@abstractplay/gameslib": "1.0.0-ci-35453221145.0", + "@abstractplay/recranks": "1.0.0-ci-35280155165.0", + "@abstractplay/renderer": "1.0.0-ci-35278756813.0", + "@aws-sdk/client-cloudwatch": "^3.1128.0", + "@aws-sdk/client-dynamodb": "^3.321.1", + "@aws-sdk/client-s3": "^3.374.0", + "@aws-sdk/client-ses": "^3.321.1", + "@aws-sdk/client-sqs": "^3.374.0", + "@aws-sdk/lib-dynamodb": "^3.321.1", + "@sparticuz/chromium": "^143.0.0", + "aws-lambda": "^1.0.7", + "fflate": "^0.8.1", + "i18next": "^22.4.15", + "ion-js": "^5.2.0", + "nanoid": "^5.1.5", + "puppeteer-core": "^24.33.0", + "stream-json": "^1.8.0", + "uuid": "^11.1.1", + "web-push": "^3.6.3" + }, + "devDependencies": { + "@abstractplay/ap-deps-tools": "^1.1.1", + "@aws-sdk/client-cloudfront": "^3.374.0", + "@aws-sdk/types": "^3.310.0", + "@types/aws-lambda": "^8.10.115", + "@types/node": "^20.19.0", + "@types/web-push": "^3.3.2", + "@typescript-eslint/eslint-plugin": "^5.59.1", + "@typescript-eslint/parser": "^5.59.1", + "esbuild": "^0.27.1", + "eslint": "^8.39.0", + "fs-extra": "^11.3.4", + "rimraf": "^6.1.2", + "serverless": "4.42.0", + "serverless-esbuild": "^1.56.1", + "serverless-scriptable-plugin": "^1.3.1", + "tsx": "^4.20.5", + "typescript": "^5.0.4", + "vite": "^6.4.1", + "vitest": "^3.2.4" + } +} diff --git a/crons/scripts/build-layers.mjs b/crons/scripts/build-layers.mjs new file mode 100644 index 00000000..79570f0c --- /dev/null +++ b/crons/scripts/build-layers.mjs @@ -0,0 +1,409 @@ +import fs from "fs-extra"; +import path from "path"; +import { execSync } from "child_process"; +import { fileURLToPath } from "url"; +import { getLockfileVersions } from "@abstractplay/ap-deps-tools/lockfile-versions"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const CRONS_ROOT = path.resolve(__dirname, ".."); +/** Workspace hoists dependencies and lockfile to the monorepo root. */ +const MONOREPO_ROOT = path.resolve(CRONS_ROOT, ".."); +const LAYER_UNZIPPED_LIMIT = 262_144_000; // 250 MiB — AWS Lambda layer limit + +/** Directory names removed when found under layer node_modules (not at repo root). */ +const PRUNE_DIR_NAMES = new Set([ + "doc", + "docs", + "example", + "examples", + "test", + "tests", + "__tests__", + "fixtures", + ".github", + "i18n", + "coverage", + "benchmark", + "bench", +]); + +/** File-name predicates for pruning under layer node_modules only. */ +const PRUNE_FILE_MATCHERS = [ + (name) => /\.md$/i.test(name), + (name) => /^README/i.test(name), + (name) => /^CHANGELOG/i.test(name), + (name) => /^HISTORY/i.test(name), + (name) => /^CONTRIBUTING/i.test(name), + (name) => /^AUTHORS/i.test(name), + (name) => /^LICENSE/i.test(name), + (name) => /^LICENCE/i.test(name), + (name) => /\.test\.js$/i.test(name), + (name) => /\.spec\.js$/i.test(name), + (name) => /\.map$/i.test(name), + (name) => /\.d\.ts$/i.test(name), + (name) => name === "tsconfig.json", + (name) => name === "jsconfig.json", + (name) => name === "Makefile", + (name) => name === ".eslintrc.js", +]; + +/** AP package roots (@abstractplay/*) — extra dirs/files stripped inside those trees. */ +const AP_PACKAGE_CRUFT_DIRS = new Set([ + "src", + "test", + "tests", + "docs", + "playground", + "scripts", + "bin", + ".github", + ".cursor", + "node_modules", +]); + +const AP_PACKAGE_CRUFT_FILE = [ + /^README/i, + /^CHANGELOG/i, + /^TODO$/i, + /\.md$/i, + /^eslint\.config\./, + /^webpack\.config\./, + /^tsconfig/, + /^serverless\.yml$/, + /^i18next-parser\.config\./, + /^\.aiexclude$/, +]; + +function shouldRemoveApPackageFile(name) { + if (name.endsWith(".map")) { + return true; + } + return AP_PACKAGE_CRUFT_FILE.some((pattern) => pattern.test(name)); +} + +function formatBytes(bytes) { + return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`; +} + +/** + * @param {string} dirPath + * @returns {Promise} + */ +async function dirSize(dirPath) { + let total = 0; + const entries = await fs.readdir(dirPath, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(dirPath, entry.name); + if (entry.isDirectory()) { + total += await dirSize(full); + } else if (entry.isFile()) { + total += (await fs.stat(full)).size; + } + } + return total; +} + +/** + * Refuse paths outside the layer root (follows symlinks/junctions). + * @param {string} layerDir + * @param {string} targetPath + * @returns {Promise} resolved absolute path safe to touch + */ +async function assertUnderLayer(layerDir, targetPath) { + const root = await fs.realpath(layerDir); + let resolved; + try { + resolved = await fs.realpath(targetPath); + } catch { + resolved = path.resolve(targetPath); + } + const relative = path.relative(root, resolved); + if (relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error( + `Refusing to touch path outside layer: ${resolved} (layer root: ${root})`, + ); + } + return resolved; +} + +/** + * Remove only when the target resolves inside layerDir. + * @param {string} layerDir + * @param {string} targetPath + */ +async function safeRemove(layerDir, targetPath) { + if (!(await fs.pathExists(targetPath))) { + return; + } + await assertUnderLayer(layerDir, targetPath); + console.log(` - Removing ${targetPath}`); + await fs.remove(targetPath); +} + +/** + * Walk layer node_modules and prune known cruft without repo-wide globs. + * @param {string} layerDir + * @param {string} nodeModulesDir + */ +async function pruneLayerNodeModules(layerDir, nodeModulesDir) { + await assertUnderLayer(layerDir, nodeModulesDir); + + async function walk(dir) { + await assertUnderLayer(layerDir, dir); + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch (err) { + console.warn(` - skip unreadable ${dir}: ${err.message}`); + return; + } + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (PRUNE_DIR_NAMES.has(entry.name)) { + await safeRemove(layerDir, fullPath); + } else if (entry.name === "@types") { + await safeRemove(layerDir, fullPath); + } else { + await walk(fullPath); + } + } else if (entry.isFile() && PRUNE_FILE_MATCHERS.some((match) => match(entry.name))) { + await safeRemove(layerDir, fullPath); + } + } + } + + if (await fs.pathExists(nodeModulesDir)) { + await walk(nodeModulesDir); + } +} + +/** + * Trim cruft inside @abstractplay/* package trees (src, tests, docs, etc.). + * @param {string} layerDir + * @param {string} pkgPath + */ +async function trimApPackage(layerDir, pkgPath) { + if (!(await fs.pathExists(pkgPath))) { + return; + } + await assertUnderLayer(layerDir, pkgPath); + + async function walk(dir, apPackageRoot) { + await assertUnderLayer(layerDir, dir); + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (apPackageRoot && AP_PACKAGE_CRUFT_DIRS.has(entry.name)) { + await safeRemove(layerDir, fullPath); + } else { + await walk(fullPath, false); + } + } else if ( + entry.isFile() + && apPackageRoot + && shouldRemoveApPackageFile(entry.name) + ) { + await safeRemove(layerDir, fullPath); + } + } + } + + await walk(pkgPath, true); +} + +/** + * Copy a package (and non-excluded deps) from project node_modules into the layer. + * @param {string} layerDir + * @param {string} packageName + * @param {string[]} excludeDeps + */ +async function syncPackageDeps(layerDir, nodeModulesDir, packageName, excludeDeps = []) { + const src = path.resolve(MONOREPO_ROOT, "node_modules", ...packageName.split("/")); + const dest = path.resolve(nodeModulesDir, ...packageName.split("/")); + + if (!(await fs.pathExists(src))) { + console.warn(`Warning: Local source for ${packageName} not found at ${src}. Skipping override.`); + return; + } + + console.log(`Syncing local code for ${packageName} into layer...`); + await safeRemove(layerDir, dest); + await fs.ensureDir(path.dirname(dest)); + await fs.copy(src, dest, { overwrite: true }); + + const internalPkg = await fs.readJson(path.join(src, "package.json")); + if (!internalPkg.dependencies) { + return; + } + + const excluded = new Set(excludeDeps); + for (const dep of Object.keys(internalPkg.dependencies)) { + if (excluded.has(dep)) { + continue; + } + const depSrc = path.resolve(MONOREPO_ROOT, "node_modules", dep); + const depDest = path.resolve(nodeModulesDir, dep); + if (await fs.pathExists(depSrc)) { + await safeRemove(layerDir, depDest); + await fs.copy(depSrc, depDest, { overwrite: true }); + } + } +} + +/** + * @param {string} layerDir + * @param {string} nodejsDir + */ +async function pruneGameslibLayer(layerDir, nodejsDir) { + console.log("Pruning renderer/chromium from gameslib layer..."); + const packagesToRemove = [ + path.join(nodejsDir, "node_modules", "@abstractplay", "renderer"), + path.join(nodejsDir, "node_modules", "@sparticuz", "chromium"), + path.join(nodejsDir, "node_modules", "puppeteer-core"), + ]; + for (const pkgPath of packagesToRemove) { + await safeRemove(layerDir, pkgPath); + } + + const gameslibDir = path.join(nodejsDir, "node_modules", "@abstractplay", "gameslib"); + for (const item of ["docs", "README.md"]) { + await safeRemove(layerDir, path.join(gameslibDir, item)); + } + + const sourceLocalesEn = path.resolve( + MONOREPO_ROOT, + "node_modules/@abstractplay/gameslib/locales/en", + ); + const targetLocalesEn = path.join(gameslibDir, "locales", "en"); + if (await fs.pathExists(sourceLocalesEn)) { + await fs.ensureDir(path.join(gameslibDir, "locales")); + await fs.copy(sourceLocalesEn, targetLocalesEn, { overwrite: true }); + console.log(" - Ensured English locale bundles in layer gameslib"); + } + + const localesDir = path.join(gameslibDir, "locales"); + if (await fs.pathExists(localesDir)) { + const localeLangs = await fs.readdir(localesDir); + for (const lang of localeLangs) { + if (lang !== "en") { + await safeRemove(layerDir, path.join(localesDir, lang)); + } + } + } +} + +/** + * @param {{ + * dir: string; + * packages: string[]; + * overridePackages?: string[]; + * excludeDeps?: string[]; + * postInstall?: (layerDir: string, nodejsDir: string, nodeModulesDir: string) => Promise; + * }} config + */ +async function createLayer(config) { + const layerDir = path.resolve(CRONS_ROOT, `.serverless/layers/${config.dir}`); + const nodejsDir = path.join(layerDir, "nodejs"); + const nodeModulesDir = path.join(nodejsDir, "node_modules"); + + console.log(`Creating ${config.dir} layer...`); + + await fs.emptyDir(layerDir); + await fs.ensureDir(nodejsDir); + + await fs.writeFile( + path.join(nodejsDir, "build-info.txt"), + `Build time: ${new Date().toISOString()}`, + ); + + const layerPackageJson = { + type: "module", + dependencies: {}, + }; + + const lockVersions = getLockfileVersions(MONOREPO_ROOT, config.packages); + + for (const pkg of config.packages) { + const version = lockVersions[pkg]; + if (!version) { + throw new Error(`Could not find ${pkg} in package-lock.json (run npm run sync-deps first)`); + } + layerPackageJson.dependencies[pkg] = version; + } + + await fs.writeJson(path.join(nodejsDir, "package.json"), layerPackageJson, { spaces: 2 }); + + const npmrcPath = path.join(MONOREPO_ROOT, ".npmrc"); + if (await fs.pathExists(npmrcPath)) { + await fs.copy(npmrcPath, path.join(nodejsDir, ".npmrc")); + } + + console.log(`Installing dependencies for ${config.dir} layer...`); + execSync("npm install --omit=dev --no-package-lock", { + cwd: nodejsDir, + stdio: "inherit", + }); + + await assertUnderLayer(layerDir, nodeModulesDir); + + for (const name of config.overridePackages ?? []) { + await syncPackageDeps(layerDir, nodeModulesDir, name, config.excludeDeps ?? []); + } + + if (config.postInstall) { + await config.postInstall(layerDir, nodejsDir, nodeModulesDir); + } + + for (const name of config.overridePackages ?? []) { + const pkgPath = path.join(nodeModulesDir, ...name.split("/")); + await trimApPackage(layerDir, pkgPath); + } + + console.log(`Aggressively pruning node_modules for ${config.dir} layer...`); + await pruneLayerNodeModules(layerDir, nodeModulesDir); + + const size = await dirSize(layerDir); + console.log(`${config.dir} layer size: ${formatBytes(size)} (${size} bytes)`); + if (size >= LAYER_UNZIPPED_LIMIT) { + throw new Error( + `${config.dir} layer exceeds the ${formatBytes(LAYER_UNZIPPED_LIMIT)} Lambda unzipped limit`, + ); + } + + console.log(`✅ ${config.dir} layer created successfully in .serverless/layers/${config.dir}`); +} + +/** @type {Array[0]>} */ +const LAYERS = [ + { + dir: "abstractplay-gameslib", + packages: ["@abstractplay/gameslib", "@abstractplay/recranks"], + overridePackages: ["@abstractplay/gameslib"], + excludeDeps: ["@abstractplay/renderer", "puppeteer-core", "@sparticuz/chromium"], + postInstall: pruneGameslibLayer, + }, + { + dir: "abstractplay-renderer", + packages: ["@abstractplay/renderer"], + overridePackages: ["@abstractplay/renderer"], + excludeDeps: ["@abstractplay/gameslib", "puppeteer-core", "@sparticuz/chromium"], + }, + { + dir: "abstractplay-chromium", + packages: ["puppeteer-core", "@sparticuz/chromium"], + }, +]; + +async function main() { + for (const layer of LAYERS) { + await createLayer(layer); + } +} + +main().catch((err) => { + console.error("Error creating layers:", err); + process.exit(1); +}); diff --git a/crons/scripts/check-apback-locales.mjs b/crons/scripts/check-apback-locales.mjs new file mode 100644 index 00000000..f61edf96 --- /dev/null +++ b/crons/scripts/check-apback-locales.mjs @@ -0,0 +1,14 @@ +import { diffApbackLocales, resolveNodeBackendRoot } from "./sync-apback-locales.mjs"; + +const { ok, mismatches } = await diffApbackLocales(); +if (ok) { + console.log(`apback locales match node-backend (${resolveNodeBackendRoot()})`); + process.exit(0); +} + +console.error("Vendored src/locales/*/apback.json is out of date with node-backend:"); +for (const m of mismatches) { + console.error(` ${m.lang}: ${m.expected} (${m.actual})`); +} +console.error("Run: npm run sync-apback-locales"); +process.exit(1); diff --git a/crons/scripts/enable-records-cdn-compress.mjs b/crons/scripts/enable-records-cdn-compress.mjs new file mode 100644 index 00000000..e2eaa5cd --- /dev/null +++ b/crons/scripts/enable-records-cdn-compress.mjs @@ -0,0 +1,48 @@ +import { + CloudFrontClient, + GetDistributionConfigCommand, + UpdateDistributionCommand, +} from "@aws-sdk/client-cloudfront"; + +const DISTRIBUTION_ID = "EM4FVU08T5188"; +const REGION = "us-east-1"; + +const dryRun = process.argv.includes("--dry-run"); + +const client = new CloudFrontClient({ region: REGION }); + +const { DistributionConfig, ETag } = await client.send( + new GetDistributionConfigCommand({ Id: DISTRIBUTION_ID }), +); + +if (!DistributionConfig?.DefaultCacheBehavior) { + throw new Error(`Distribution ${DISTRIBUTION_ID} has no default cache behavior`); +} + +const updated = { + ...DistributionConfig, + DefaultCacheBehavior: { + ...DistributionConfig.DefaultCacheBehavior, + Compress: true, + }, +}; + +if (DistributionConfig.DefaultCacheBehavior.Compress === true) { + console.log(`Distribution ${DISTRIBUTION_ID}: Compress already enabled on default behavior`); + process.exit(0); +} + +if (dryRun) { + console.log(`[dry-run] Would enable Compress on distribution ${DISTRIBUTION_ID}`); + process.exit(0); +} + +await client.send( + new UpdateDistributionCommand({ + Id: DISTRIBUTION_ID, + IfMatch: ETag, + DistributionConfig: updated, + }), +); + +console.log(`Enabled Compress objects automatically on distribution ${DISTRIBUTION_ID}`); diff --git a/crons/scripts/ensure-apback-locales.mjs b/crons/scripts/ensure-apback-locales.mjs new file mode 100644 index 00000000..28cd8c30 --- /dev/null +++ b/crons/scripts/ensure-apback-locales.mjs @@ -0,0 +1,35 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { APBACK_LANGS, syncApbackLocales } from "./sync-apback-locales.mjs"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const TARGET_ROOT = path.join(__dirname, "..", "src", "locales"); + +async function hasAllLocales() { + for (const lang of APBACK_LANGS) { + try { + await fs.access(path.join(TARGET_ROOT, lang, "apback.json")); + } catch { + return false; + } + } + return true; +} + +if (await hasAllLocales()) { + process.exit(0); +} + +try { + await syncApbackLocales({ write: true }); + console.log("Generated src/locales from node-backend"); +} catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(message); + console.error( + "Install apback copy with: npm run sync-apback-locales " + + "(clone node-backend as ../node-backend or set NODE_BACKEND_ROOT)", + ); + process.exit(1); +} diff --git a/crons/scripts/index.html b/crons/scripts/index.html new file mode 100644 index 00000000..f46d478b --- /dev/null +++ b/crons/scripts/index.html @@ -0,0 +1,348 @@ + + + + + + + + + + + + Abstract Play Records + + +
+
+

Abstract Play Records

+

Game reports adhere to a schema, available in the Abstract Play Records and Rankings repository.

+ +
+ +
+

All JSON records

+
+ + + + + + + + + + + +
FileSizeLast modified
+
+ + + + diff --git a/crons/scripts/preview-inactive-challenge-cleanup.ts b/crons/scripts/preview-inactive-challenge-cleanup.ts new file mode 100644 index 00000000..3d99bfbc --- /dev/null +++ b/crons/scripts/preview-inactive-challenge-cleanup.ts @@ -0,0 +1,52 @@ +/** + * Read-only preview of inactive-challenge-cleanup candidates (no DynamoDB writes). + * + * Usage: npm run preview-inactive-challenges -- --stage prod [--days 14] + */ +import { parseArgs } from "node:util"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb"; +import { discoverInactiveIssuerChallenges } from "../src/lib/inactiveChallengeDiscovery.js"; + +const { values } = parseArgs({ + options: { + stage: { type: "string", default: "dev" }, + days: { type: "string", default: "14" }, + }, +}); + +const stage = values.stage ?? "dev"; +const days = Number(values.days ?? "14"); +if (!Number.isFinite(days) || days <= 0) { + throw new Error("--days must be a positive number"); +} + +const tableName = `abstract-play-${stage}`; +const profile = stage === "prod" ? "AbstractPlayProd" : "AbstractPlayDev"; +process.env.AWS_PROFILE = profile; + +const inactiveBeforeMs = Date.now() - days * 24 * 60 * 60 * 1000; +const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({ region: "us-east-1" })); + +const discovery = await discoverInactiveIssuerChallenges(ddb, tableName, inactiveBeforeMs); + +console.log( + JSON.stringify( + { + tableName, + inactiveDays: days, + inactiveBeforeMs, + inactiveUsers: discovery.inactiveUsers, + openChallengesScanned: discovery.openChallengesScanned, + candidateCount: discovery.candidates.length, + candidates: discovery.candidates.map((c) => ({ + kind: c.kind, + metaGame: c.metaGame, + id: c.id, + issuerId: c.issuerId, + })), + }, + null, + 2, + ), +); diff --git a/crons/scripts/run-records-pipeline.mjs b/crons/scripts/run-records-pipeline.mjs new file mode 100644 index 00000000..2101e693 --- /dev/null +++ b/crons/scripts/run-records-pipeline.mjs @@ -0,0 +1,40 @@ +import { execSync } from "node:child_process"; + +/** + * Invoke batch record pipeline Lambdas in dependency order (prod/dev). + * Does not run dumpdb (async export) or SQS-driven workers. + * + * Usage: npm run run-records-pipeline -- --stage prod + */ + +function readStage() { + const idx = process.argv.indexOf("--stage"); + if (idx >= 0 && process.argv[idx + 1]) { + return process.argv[idx + 1]; + } + return "dev"; +} + +const stage = readStage(); +const functions = [ + "records", + "records-ttm", + "records-move-times", + "records-cooccur", + "records-rec-analytics", + "tournament-data", + "records-manifest", + "summarize", + "player-summary-fanout", + "records-manifest", +]; + +for (const name of functions) { + console.log(`\n>>> serverless invoke -f ${name} --stage ${stage}`); + execSync(`npx serverless invoke -f ${name} --stage ${stage}`, { + stdio: "inherit", + env: process.env, + }); +} + +console.log("\nPipeline invoke sequence finished."); diff --git a/crons/scripts/smoke-layer-modules.mjs b/crons/scripts/smoke-layer-modules.mjs new file mode 100644 index 00000000..0ed6f9b1 --- /dev/null +++ b/crons/scripts/smoke-layer-modules.mjs @@ -0,0 +1,68 @@ +import path from "path"; +import { fileURLToPath, pathToFileURL } from "url"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +/** + * @param {string} layerDir + * @param {string[]} pkgSegments + */ +async function importLayerEntry(layerDir, pkgSegments) { + const entry = path.join( + ROOT, + ".serverless", + "layers", + layerDir, + "nodejs", + "node_modules", + ...pkgSegments, + "build", + "index.js", + ); + return import(pathToFileURL(entry).href); +} + +/** + * @param {string} layerDir + * @param {string} pkgName + */ +async function assertLayerPackagePresent(layerDir, pkgName) { + const pkgPath = path.join( + ROOT, + ".serverless", + "layers", + layerDir, + "nodejs", + "node_modules", + ...pkgName.split("/"), + "package.json", + ); + const { access } = await import("node:fs/promises"); + await access(pkgPath); +} + +try { + const gl = await importLayerEntry("abstractplay-gameslib", ["@abstractplay", "gameslib"]); + if (!gl.gameinfo || typeof gl.GameFactory !== "function") { + throw new Error("@abstractplay/gameslib missing expected exports"); + } + + const rr = await importLayerEntry("abstractplay-gameslib", ["@abstractplay", "recranks"]); + if (typeof rr.Glicko2 !== "function" || typeof rr.ELOBasic !== "function") { + throw new Error("@abstractplay/recranks missing expected exports"); + } + + const renderer = await importLayerEntry("abstractplay-renderer", ["@abstractplay", "renderer"]); + if (typeof renderer.addPrefix !== "function") { + throw new Error("@abstractplay/renderer addPrefix missing after layer import"); + } + + await assertLayerPackagePresent("abstractplay-chromium", "puppeteer-core"); + await assertLayerPackagePresent("abstractplay-chromium", "@sparticuz/chromium"); + + console.log("smoke-layer-modules: gameslib + recranks + renderer + chromium layers OK"); +} catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(`smoke-layer-modules: ${message}`); + process.exit(1); +} diff --git a/crons/scripts/sync-apback-locales.mjs b/crons/scripts/sync-apback-locales.mjs new file mode 100644 index 00000000..c1da6235 --- /dev/null +++ b/crons/scripts/sync-apback-locales.mjs @@ -0,0 +1,142 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, ".."); +const TARGET_ROOT = path.join(ROOT, "src", "locales"); + +/** Must match node-backend `locales/` layout. */ +export const APBACK_LANGS = ["en", "fr", "de", "it", "es-US", "pt", "ta"]; + +export function resolveNodeBackendRoot() { + const fromEnv = process.env.NODE_BACKEND_ROOT; + if (fromEnv) { + return path.resolve(fromEnv); + } + return path.resolve(ROOT, ".."); +} + +function sha256(text) { + return crypto.createHash("sha256").update(text, "utf8").digest("hex"); +} + +/** + * @param {{ write?: boolean }} opts + * @returns {Promise<{ sourceRoot: string; copied: string[]; removed: string[] }>} + */ +export async function syncApbackLocales(opts = {}) { + const write = opts.write !== false; + const sourceRoot = resolveNodeBackendRoot(); + const sourceLocales = path.join(sourceRoot, "locales"); + + try { + await fs.access(sourceLocales); + } catch { + throw new Error( + `node-backend locales not found at ${sourceLocales}. ` + + "Set NODE_BACKEND_ROOT or clone AbstractPlay/node-backend as a sibling.", + ); + } + + const copied = []; + for (const lang of APBACK_LANGS) { + const src = path.join(sourceLocales, lang, "apback.json"); + const dest = path.join(TARGET_ROOT, lang, "apback.json"); + const body = await fs.readFile(src, "utf8"); + JSON.parse(body); + if (write) { + await fs.mkdir(path.dirname(dest), { recursive: true }); + await fs.writeFile(dest, body.endsWith("\n") ? body : `${body}\n`, "utf8"); + } + copied.push(dest); + } + + const removed = []; + if (write) { + let entries; + try { + entries = await fs.readdir(TARGET_ROOT, { withFileTypes: true }); + } catch { + entries = []; + } + for (const entry of entries) { + if (!entry.isDirectory()) { + continue; + } + if (APBACK_LANGS.includes(entry.name)) { + continue; + } + const langDir = path.join(TARGET_ROOT, entry.name); + await fs.rm(langDir, { recursive: true, force: true }); + removed.push(langDir); + } + } + + return { sourceRoot, copied, removed }; +} + +/** + * @returns {Promise<{ ok: boolean; mismatches: { lang: string; expected: string; actual: string }[] }>} + */ +export async function diffApbackLocales() { + const sourceRoot = resolveNodeBackendRoot(); + const sourceLocales = path.join(sourceRoot, "locales"); + const mismatches = []; + + for (const lang of APBACK_LANGS) { + const src = path.join(sourceLocales, lang, "apback.json"); + const dest = path.join(TARGET_ROOT, lang, "apback.json"); + let expected; + let actual; + try { + expected = await fs.readFile(src, "utf8"); + } catch { + mismatches.push({ lang, expected: "missing source", actual: dest }); + continue; + } + try { + actual = await fs.readFile(dest, "utf8"); + } catch { + mismatches.push({ lang, expected: sha256(expected), actual: "missing vendored file" }); + continue; + } + const normExpected = expected.endsWith("\n") ? expected : `${expected}\n`; + const normActual = actual.endsWith("\n") ? actual : `${actual}\n`; + if (sha256(normExpected) !== sha256(normActual)) { + mismatches.push({ lang, expected: "out of date", actual: dest }); + } + } + + let entries; + try { + entries = await fs.readdir(TARGET_ROOT, { withFileTypes: true }); + } catch { + entries = []; + } + for (const entry of entries) { + if (entry.isDirectory() && !APBACK_LANGS.includes(entry.name)) { + mismatches.push({ + lang: entry.name, + expected: "removed", + actual: path.join(TARGET_ROOT, entry.name), + }); + } + } + + return { ok: mismatches.length === 0, mismatches }; +} + +const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); + +if (isMain) { + const result = await syncApbackLocales({ write: true }); + console.log(`Synced apback locales from ${result.sourceRoot}`); + for (const file of result.copied) { + console.log(` ${path.relative(ROOT, file)}`); + } + for (const dir of result.removed) { + console.log(` removed ${path.relative(ROOT, dir)}`); + } +} diff --git a/crons/serverless.yml b/crons/serverless.yml new file mode 100644 index 00000000..0cc572af --- /dev/null +++ b/crons/serverless.yml @@ -0,0 +1,649 @@ +service: abstract-play-backend-crons +org: abstractplay + +frameworkVersion: "4" + +build: + esbuild: false + +custom: + scheduleEnabled: + prod: true + dev: false + # node-backend exports abstract-play-${stage}-OpsAlertsTopicArn (abstractplay-ops-alerts-${stage}) + opsAlertsTopicImport: abstract-play-${self:provider.stage}-OpsAlertsTopicArn + recbucket: + prod: "records.abstractplay.com" + dev: "records.abstractplay.com" + dumpbucket: "abstractplay-db-dump" + opsbucket: "private-ops-153672715141-us-east-1-an" + renderBucketName: "thumbnails-prerender-${self:provider.stage}" + rendererCdnUrl: "https://renderer.dev.abstractplay.com/APRender.min.js" + esbuild: + tsconfig: "./tsconfig.json" + bundle: true + sourcemap: false # Sourcemaps are large and not needed in production + format: "esm" + outExtension: + ".js": ".mjs" + platform: "node" + target: "node24" + external: + - "@abstractplay/gameslib" + - "@abstractplay/recranks" + - "@abstractplay/renderer" + - "@sparticuz/chromium" + - "puppeteer-core" + - "@aws-sdk/*" + - "@smithy/*" + - "buffer" + - "fs" + - "stream-json" + - "stream-chain" + - "web-push" + exclude: + - "@abstractplay/gameslib" + - "@abstractplay/recranks" + - "@abstractplay/renderer" + - "@sparticuz/chromium" + - "puppeteer-core" + scriptable: + hooks: + before:package:createDeploymentArtifacts: npm run build:layers + +params: + dev: + profile: AbstractPlayDev + prod: + profile: AbstractPlayProd + +plugins: + - serverless-esbuild + - serverless-scriptable-plugin + +provider: + name: aws + runtime: nodejs24.x + versionFunctions: false + stage: ${opt:stage, "dev"} + profile: ${param:profile} + region: us-east-1 + memorySize: 1024 + logs: + retentionInDays: 30 # set to any valid retention period (1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653) + environment: + ABSTRACT_PLAY_TABLE: abstract-play-${self:provider.stage} + iam: + role: + statements: + - Effect: Allow + Action: + - dynamodb:Query + - dynamodb:Scan + - dynamodb:GetItem + - dynamodb:BatchGetItem + - dynamodb:PutItem + - dynamodb:UpdateItem + - dynamodb:DeleteItem + - dynamodb:ExportTableToPointInTime + - dynamodb:BatchWriteItem + - sqs:SendMessage + - sqs:SendMessageBatch + - sqs:ReceiveMessage + - sqs:DeleteMessage + - sqs:GetQueueAttributes + - sqs:GetQueueUrl + - ses:SendEmail + - ses:SendRawEmail + - cloudwatch:PutMetricData + Resource: "*" + - Effect: "Allow" + Action: + - "s3:ListBucket" + Resource: + - "arn:aws:s3:::abstractplay-db-dump" + - "arn:aws:s3:::records.abstractplay.com" + - "arn:aws:s3:::${self:custom.opsbucket}" + - "arn:aws:s3:::${self:custom.renderBucketName}" + - Effect: "Allow" + Action: + - "s3:GetObject" + Resource: + - "arn:aws:s3:::abstractplay-db-dump/*" + - "arn:aws:s3:::records.abstractplay.com/*" + - "arn:aws:s3:::${self:custom.renderBucketName}/*" + - "arn:aws:s3:::thumbnails.abstractplay.com/*" + - Effect: "Allow" + Action: + - "s3:GetObject" + Resource: + - "arn:aws:s3:::${self:custom.opsbucket}/*" + - Effect: "Allow" + Action: + - "s3:PutObject" + - "s3:DeleteObject" + Resource: + - "arn:aws:s3:::records.abstractplay.com/*" + - "arn:aws:s3:::thumbnails.abstractplay.com/*" + - "arn:aws:s3:::abstractplay-db-dump/*" + - "arn:aws:s3:::${self:custom.opsbucket}/*" + - "arn:aws:s3:::${self:custom.renderBucketName}/*" + +functions: + dumpdb: + handler: src/functions/dumpdb.handler + description: Triggers a full export of production DB to S3 + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-dumpdb + description: Triggers a full export of production DB to S3 + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # midnight UTC daily + schedule: cron(0 0 * * ? *) + + records: + handler: src/functions/records.handler + description: Generates static lists of game records + timeout: 900 + memorySize: 10240 + logRetentionInDays: 30 + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-records + description: Generates static lists of game records + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (uses latest completed dump by LastModified) + schedule: cron(0 3 * * ? *) + + records-ttm: + handler: src/functions/records-ttm.handler + description: Generates static time-to-move results of each game + timeout: 900 + memorySize: 10240 + logRetentionInDays: 30 + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-records-ttm + description: Generates static time-to-move results of each game + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (uses latest completed dump by LastModified) + schedule: cron(0 3 * * ? *) + + records-move-times: + handler: src/functions/records-move-times.handler + description: Generates list of moves made in past 180 days + timeout: 900 + memorySize: 10240 + logRetentionInDays: 30 + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-records-move-times + description: Generates list of moves made in past 180 days + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (uses latest completed dump by LastModified) + schedule: cron(0 3 * * ? *) + + records-cooccur: + handler: src/functions/records-cooccur.handler + description: Builds PMI game co-occurrence matrix for recommendations + timeout: 900 + memorySize: 10240 + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-records-cooccur + description: Builds PMI game co-occurrence matrix for recommendations + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (uses latest completed dump by LastModified) + schedule: cron(0 3 * * ? *) + + dashboard-cruft-cleanup: + handler: src/functions/dashboard-cruft-cleanup.handler + description: Prunes stale dashboard index cruft for long-inactive users (S3 dump candidates + live DDB confirm) + timeout: 900 + memorySize: 1024 + logRetentionInDays: 30 + environment: + DASHBOARD_CRUFT_BATCH_SIZE: 75 + ABANDONED_ACCOUNT_INACTIVE_MS: 31536000000 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-dashboard-cruft-cleanup + description: Daily abandoned-account dashboard cruft cleanup + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (after dumpdb export; uses latest completed ION dump for candidates) + schedule: cron(0 3 * * ? *) + + records-rec-analytics: + handler: src/functions/records-rec-analytics.handler + description: Aggregates recommendation impression events to private ops S3 + timeout: 900 + memorySize: 1024 + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-records-rec-analytics + description: Nightly recommendation impression funnel analytics + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (live DynamoDB scan of RECOMMENDS# events) + schedule: cron(0 3 * * ? *) + + records-manifest: + handler: src/functions/records-manifest.handler + description: Generates manifest file for records bucket + timeout: 900 + memorySize: 10240 + logRetentionInDays: 30 + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-records-manifest-early + description: Manifest after records batch outputs + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 4am UTC daily + schedule: cron(0 4 * * ? *) + - eventBridge: + name: abstractplay-${self:provider.stage}-records-manifest-late + description: Manifest after summarize and player-summary fan-out + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 7:30am UTC daily (summarize 6:00, fan-out 6:15, workers finish before this) + schedule: cron(30 7 * * ? *) + + summarize: + handler: src/functions/summarize.handler + description: Summarize generated game reports + timeout: 900 + memorySize: 5120 + layers: + - !Ref AbstractplayGameslibLambdaLayer + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-summarize + description: Summarize generated game reports + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 6am UTC daily (giving plenty of time for the record generation to complete) + schedule: cron(0 6 * * ? *) + + thumbnails: + handler: src/functions/thumbnails.handler + description: Generates random thumbnails daily from latest DB dump + timeout: 300 + memorySize: 5120 + logRetentionInDays: 7 + environment: + RENDER_BUCKET: ${self:custom.renderBucketName} + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-thumbnails + description: Generates random thumbnails daily + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 6am UTC daily (independent of summarize; reads ION dump directly) + schedule: cron(0 6 * * ? *) + + thumbnails-verify: + handler: src/functions/thumbnails-verify.handler + description: Verifies thumbnail SVG Last-Modified is not older than JSON after render pipeline + timeout: 300 + memorySize: 512 + logRetentionInDays: 7 + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-thumbnails-verify + description: Checks JSON/SVG freshness one hour after thumbnails cron + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 7am UTC daily (after thumbnails + render pipeline) + schedule: cron(0 7 * * ? *) + + s3-to-sqs: + handler: src/functions/s3-to-sqs.handler + description: Triggered when new objects are written to prerender S3, pushes job info to SQS + timeout: 30 + memorySize: 512 + logRetentionInDays: 7 + environment: + SQS_URL: !Ref RenderQueue + TARGET_QUEUE_ARN: !GetAtt RenderQueue.Arn + RENDER_BUCKET: ${self:custom.renderBucketName} + events: + - s3: + bucket: ${self:custom.renderBucketName} + existing: true + event: s3:ObjectCreated:* + + sqs-to-render: + handler: src/functions/sqs-to-render.handler + description: Consumes SQS messages, renders SVG, writes to output bucket, deletes prerender original + timeout: 300 + memorySize: 2048 + logRetentionInDays: 7 + environment: + RENDER_BUCKET: ${self:custom.renderBucketName} + RENDERER_CDN_URL: ${self:custom.rendererCdnUrl} + layers: + - !Ref AbstractplayChromiumLambdaLayer + - !Ref AbstractplayGameslibLambdaLayer + - !Ref AbstractplayRendererLambdaLayer + events: + - sqs: + arn: !GetAtt RenderQueue.Arn + + player-summary-fanout: + handler: src/functions/player-summary-fanout.handler + description: Enqueues per-player summary slice writes after summarize + timeout: 300 + memorySize: 1024 + layers: + - !Ref AbstractplayGameslibLambdaLayer + logRetentionInDays: 30 + environment: + PLAYER_SUMMARY_QUEUE_URL: !Ref PlayerSummaryQueue + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-player-summary-fanout + description: Fan out player summary slice writes via SQS + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 6:15am UTC daily (after summarize at 6:00) + schedule: cron(15 6 * * ? *) + + rating-change-notifications: + handler: src/functions/rating-change-notifications.handler + description: Batch ratingChange in-app notifications after summarize Glicko update + timeout: 300 + memorySize: 1024 + layers: + - !Ref AbstractplayGameslibLambdaLayer + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-rating-change-notifications + description: Diff batch Glicko ratings and enqueue ratingChange notifications + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 6:20am UTC daily (after summarize at 6:00 and player-summary-fanout at 6:15) + schedule: cron(20 6 * * ? *) + + player-summary-worker: + handler: src/functions/player-summary-worker.handler + description: Writes one player summary slice per SQS message + timeout: 30 + memorySize: 256 + reservedConcurrency: 25 + logRetentionInDays: 30 + events: + - sqs: + arn: !GetAtt PlayerSummaryQueue.Arn + batchSize: 5 + maximumBatchingWindow: 5 + + tournament-data: + handler: src/functions/tournament-data.handler + description: Summarizes tournament data + timeout: 900 + memorySize: 10240 + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-tournament-data + description: Summarizes tournament data + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily (uses latest completed dump by LastModified) + schedule: cron(0 3 * * ? *) + + starttournaments: + handler: src/functions/starttournaments.handler + description: Checks if any tournaments are ready to start (and starts or cancels them) + timeout: 600 + logRetentionInDays: 30 + layers: + - !Ref AbstractplayGameslibLambdaLayer + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-starttournaments + description: Checks if any tournaments are ready to start (and starts or cancels them) + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 10am and 10pm UTC every day + schedule: cron(0 10,22 * * ? *) + + standingchallenges: + handler: src/functions/standingchallenges.handler + description: Manages preset standing challenge requests + timeout: 600 + logRetentionInDays: 30 + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-standingchallenges + description: Manages preset standing challenge requests + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # Midnight and noon UTC every day + schedule: cron(0 0,12 * * ? *) + + inactive-challenge-cleanup: + handler: src/functions/inactive-challenge-cleanup.handler + description: Revoke challenges from issuers inactive ≥14 days; pause REALSTANDING presets + timeout: 300 + memorySize: 512 + logRetentionInDays: 30 + layers: + - !Ref AbstractplayGameslibLambdaLayer + environment: + INACTIVE_CHALLENGE_MS: 1209600000 + VAPID_PUBLIC_KEY: ${env:VAPID_PUBLIC_KEY, ''} + VAPID_PRIVATE_KEY: ${env:VAPID_PRIVATE_KEY, ''} + events: + - eventBridge: + name: abstractplay-${self:provider.stage}-inactive-challenge-cleanup + description: Revoke challenges from inactive issuers nightly + enabled: ${self:custom.scheduleEnabled.${self:provider.stage}} + # 3am UTC daily + schedule: cron(0 3 * * ? *) + +resources: + Conditions: + OpsAlertsEnabled: + Fn::Equals: + - ${self:provider.stage} + - prod + + Resources: + RecordsErrorsAlarm: + Type: AWS::CloudWatch::Alarm + Condition: OpsAlertsEnabled + Properties: + AlarmName: abstractplay-crons-records-errors-${self:provider.stage} + AlarmDescription: records Lambda errors (including init failures) + Namespace: AWS/Lambda + MetricName: Errors + Dimensions: + - Name: FunctionName + Value: !Ref RecordsLambdaFunction + Statistic: Sum + Period: 60 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + OKActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + + SummarizeErrorsAlarm: + Type: AWS::CloudWatch::Alarm + Condition: OpsAlertsEnabled + Properties: + AlarmName: abstractplay-crons-summarize-errors-${self:provider.stage} + AlarmDescription: summarize Lambda errors (including init failures) + Namespace: AWS/Lambda + MetricName: Errors + Dimensions: + - Name: FunctionName + Value: !Ref SummarizeLambdaFunction + Statistic: Sum + Period: 60 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + OKActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + + SqsToRenderErrorsAlarm: + Type: AWS::CloudWatch::Alarm + Condition: OpsAlertsEnabled + Properties: + AlarmName: abstractplay-crons-sqs-to-render-errors-${self:provider.stage} + AlarmDescription: sqs-to-render Lambda errors (thumbnail SVG generation) + Namespace: AWS/Lambda + MetricName: Errors + Dimensions: + - Name: FunctionName + Value: !Ref SqsDashtoDashrenderLambdaFunction + Statistic: Sum + Period: 60 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + OKActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + + ThumbnailsVerifyErrorsAlarm: + Type: AWS::CloudWatch::Alarm + Condition: OpsAlertsEnabled + Properties: + AlarmName: abstractplay-crons-thumbnails-verify-errors-${self:provider.stage} + AlarmDescription: thumbnails-verify Lambda errors (JSON/SVG Last-Modified mismatch) + Namespace: AWS/Lambda + MetricName: Errors + Dimensions: + - Name: FunctionName + Value: !Ref ThumbnailsDashverifyLambdaFunction + Statistic: Sum + Period: 60 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + OKActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + + ThumbnailRenderFailureAlarm: + Type: AWS::CloudWatch::Alarm + Condition: OpsAlertsEnabled + Properties: + AlarmName: abstractplay-crons-thumbnail-render-failure-${self:provider.stage} + AlarmDescription: Custom RenderFailure metric from sqs-to-render + Namespace: AbstractPlay/Thumbnails + MetricName: RenderFailure + Statistic: Sum + Period: 300 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + OKActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + + PlayerSummaryDeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + MessageRetentionPeriod: 1209600 + + PlayerSummaryQueue: + Type: AWS::SQS::Queue + Properties: + VisibilityTimeout: 60 + RedrivePolicy: + deadLetterTargetArn: !GetAtt PlayerSummaryDeadLetterQueue.Arn + maxReceiveCount: 3 + + RenderDeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: abstractplay-render-dlq-${self:provider.stage} + MessageRetentionPeriod: 1209600 + + RenderQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: abstractplay-render-queue-${self:provider.stage} + VisibilityTimeout: 900 + MessageRetentionPeriod: 86400 + RedrivePolicy: + deadLetterTargetArn: !GetAtt RenderDeadLetterQueue.Arn + maxReceiveCount: 3 + + RenderDlqAlarm: + Type: AWS::CloudWatch::Alarm + Condition: OpsAlertsEnabled + Properties: + AlarmName: abstractplay-crons-render-dlq-${self:provider.stage} + AlarmDescription: Messages in render queue dead-letter queue + Namespace: AWS/SQS + MetricName: ApproximateNumberOfMessagesVisible + Dimensions: + - Name: QueueName + Value: abstractplay-render-dlq-${self:provider.stage} + Statistic: Maximum + Period: 300 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + OKActions: + - Fn::ImportValue: ${self:custom.opsAlertsTopicImport} + + Outputs: + RenderQueueArn: + Value: !GetAtt RenderQueue.Arn + Export: + Name: RenderQueueArn-${self:provider.stage} + RenderQueueUrl: + Value: !Ref RenderQueue + Export: + Name: RenderQueueUrl-${self:provider.stage} + +layers: + abstractplayGameslib: + path: .serverless/layers/abstractplay-gameslib + name: abstractplay-gameslib-${sls:stage} + description: "Layer for @abstractplay/gameslib" + compatibleRuntimes: + - nodejs24.x + retain: false + abstractplayChromium: + path: .serverless/layers/abstractplay-chromium + name: abstractplay-chromium-${sls:stage} + description: "Layer for puppeteer-core and @sparticuz/chromium" + compatibleRuntimes: + - nodejs24.x + retain: false + abstractplayRenderer: + path: .serverless/layers/abstractplay-renderer + name: abstractplay-renderer-${sls:stage} + description: "Layer for @abstractplay/renderer and its runtime dependencies" + compatibleRuntimes: + - nodejs24.x + retain: false diff --git a/crons/src/constants/recordsBucket.ts b/crons/src/constants/recordsBucket.ts new file mode 100644 index 00000000..77fb0c2e --- /dev/null +++ b/crons/src/constants/recordsBucket.ts @@ -0,0 +1,9 @@ +export const REC_BUCKET = "records.abstractplay.com"; + +export const SUMMARY_MONOLITH_KEY = "_summary.json"; +export const SUMMARY_SITE_KEY = "_summary-site.json"; +export const SUMMARY_PLAYERS_KEY = "_summary-players.json"; +export const SUMMARY_RATINGS_KEY = "_summary-ratings.json"; +export const PLAYER_SUMMARY_MANIFEST_KEY = "_summary-player-manifest.json"; +export const RATINGS_NOTIFICATION_SNAPSHOT_KEY = "_ratings-notification-snapshot.json"; +export const PLAYER_SUMMARY_KEY_PATTERN = "player/{userId}-summary.json"; diff --git a/crons/src/functions/dashboard-cruft-cleanup.ts b/crons/src/functions/dashboard-cruft-cleanup.ts new file mode 100644 index 00000000..f627d15b --- /dev/null +++ b/crons/src/functions/dashboard-cruft-cleanup.ts @@ -0,0 +1,136 @@ +'use strict'; + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient, GetCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; +import { S3Client } from '@aws-sdk/client-s3'; +import type { Handler } from 'aws-lambda'; +import { cleanupUserDashboardCruft } from '../utils/dashboardCruftCleanup.js'; +import { + collectUserCandidatesFromDump, + findLatestDumpUid, + listDumpBucketObjects, +} from '../utils/dumpExport.js'; + +const REGION = 'us-east-1'; +const DEFAULT_INACTIVE_MS = 365 * 24 * 60 * 60 * 1000; +const DEFAULT_BATCH_SIZE = 75; + +type SkipReason = + | 'stale_dump' + | 'already_cleaned' + | 'active_since_dump' + | 'bot' + | 'no_cruft' + | 'error'; + +type Summary = { + candidates: number; + processed: number; + skipped: Record; + cleanedUsers: string[]; + errors: { userId: string; message: string }[]; +}; + +async function isBotId( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'BOT', sk: userId }, + ProjectionExpression: '#pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + })); + return data.Item !== undefined; +} + +export const handler: Handler = async () => { + const tableName = process.env.ABSTRACT_PLAY_TABLE; + if (!tableName) { + throw new Error('ABSTRACT_PLAY_TABLE is required'); + } + + const inactiveMs = Number(process.env.ABANDONED_ACCOUNT_INACTIVE_MS ?? DEFAULT_INACTIVE_MS); + const batchSize = Number(process.env.DASHBOARD_CRUFT_BATCH_SIZE ?? DEFAULT_BATCH_SIZE); + const now = Date.now(); + const inactiveBeforeMs = now - inactiveMs; + + const s3 = new S3Client({ region: REGION }); + const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({ region: REGION })); + + const summary: Summary = { + candidates: 0, + processed: 0, + skipped: { + stale_dump: 0, + already_cleaned: 0, + active_since_dump: 0, + bot: 0, + no_cruft: 0, + error: 0, + }, + cleanedUsers: [], + errors: [], + }; + + const allContents = await listDumpBucketObjects(s3); + const uid = findLatestDumpUid(allContents); + const candidates = await collectUserCandidatesFromDump(s3, allContents, uid, inactiveBeforeMs); + summary.candidates = candidates.length; + console.log(`dashboard-cruft-cleanup: ${candidates.length} dump candidates from export ${uid}`); + + for (const userId of candidates.slice(0, batchSize)) { + try { + const user = await ddb.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: userId }, + ProjectionExpression: 'lastSeen, cleaned', + })); + if (user.Item === undefined) { + summary.skipped.stale_dump += 1; + continue; + } + if (user.Item.cleaned === true) { + summary.skipped.already_cleaned += 1; + continue; + } + const lastSeen = user.Item.lastSeen; + if (typeof lastSeen !== 'number' || lastSeen >= inactiveBeforeMs) { + summary.skipped.active_since_dump += 1; + continue; + } + if (await isBotId(ddb, tableName, userId)) { + summary.skipped.bot += 1; + continue; + } + + const stats = await cleanupUserDashboardCruft(ddb, tableName, userId, now); + if (stats.recentCompletedDeleted === 0 && stats.userGameDeleted === 0) { + summary.skipped.no_cruft += 1; + continue; + } + + await ddb.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: userId }, + UpdateExpression: 'SET cleaned = :true', + ExpressionAttributeValues: { ':true': true }, + })); + + summary.processed += 1; + summary.cleanedUsers.push(userId); + console.log(`dashboard-cruft-cleanup: cleaned ${userId}`, stats); + } catch (error) { + summary.skipped.error += 1; + summary.errors.push({ + userId, + message: error instanceof Error ? error.message : String(error), + }); + console.error(`dashboard-cruft-cleanup: error for ${userId}`, error); + } + } + + console.log('dashboard-cruft-cleanup summary', summary); + return summary; +}; diff --git a/crons/src/functions/dumpdb.ts b/crons/src/functions/dumpdb.ts new file mode 100644 index 00000000..ddc2e59b --- /dev/null +++ b/crons/src/functions/dumpdb.ts @@ -0,0 +1,111 @@ +'use strict'; + +import { DynamoDBClient, ExportTableToPointInTimeCommand, type ExportTableToPointInTimeInput } from "@aws-sdk/client-dynamodb"; +import { S3Client, ListObjectsV2Command, DeleteObjectsCommand, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; + +const REGION = "us-east-1"; +const DUMP_BUCKET = "abstractplay-db-dump"; +const EXPORT_PREFIX = "AWSDynamoDB/"; +const RETENTION_DAYS = 7; +const ddbClient = new DynamoDBClient({ region: REGION }); +const s3Client = new S3Client({ region: REGION }); + +async function listAllObjects(bucket: string, prefix: string): Promise<_Object[]> { + const command = new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix }); + const allContents: _Object[] = []; + let isTruncated = true; + + while (isTruncated) { + const { Contents, IsTruncated, NextContinuationToken } = await s3Client.send(command); + if (Contents === undefined) { + throw new Error(`Could not list the bucket contents`); + } + allContents.push(...Contents); + isTruncated = IsTruncated || false; + command.input.ContinuationToken = NextContinuationToken; + } + + return allContents; +} + +async function pruneOldExports(retentionDays: number): Promise { + const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000; + const allContents = await listAllObjects(DUMP_BUCKET, EXPORT_PREFIX); + + const exportTimes = new Map(); + for (const obj of allContents) { + if (!obj.Key?.endsWith("manifest-summary.json")) { + continue; + } + const match = obj.Key.match(/^AWSDynamoDB\/([^/]+)\/manifest-summary\.json$/); + if (match !== null && obj.LastModified !== undefined) { + exportTimes.set(match[1], obj.LastModified.getTime()); + } + } + + if (exportTimes.size === 0) { + console.log("No exports found to prune"); + return; + } + + const latestUid = [...exportTimes.entries()] + .sort((a, b) => b[1] - a[1])[0]![0]; + + const keysToDelete: string[] = []; + for (const obj of allContents) { + const match = obj.Key?.match(/^AWSDynamoDB\/([^/]+)\//); + if (match === null || match === undefined) { + continue; + } + const uid = match[1]; + const exportTime = exportTimes.get(uid); + if (exportTime === undefined || uid === latestUid || exportTime >= cutoff) { + continue; + } + keysToDelete.push(obj.Key!); + } + + if (keysToDelete.length === 0) { + console.log(`No exports older than ${retentionDays} days to delete`); + return; + } + + for (let i = 0; i < keysToDelete.length; i += 1000) { + const batch = keysToDelete.slice(i, i + 1000); + const response = await s3Client.send(new DeleteObjectsCommand({ + Bucket: DUMP_BUCKET, + Delete: { Objects: batch.map(Key => ({ Key })) }, + })); + console.log(`Deleted ${response.Deleted?.length ?? 0} objects`); + if (response.Errors !== undefined && response.Errors.length > 0) { + console.error(`Delete errors:\n${JSON.stringify(response.Errors, null, 2)}`); + } + } + + console.log(`Pruned ${keysToDelete.length} objects from exports older than ${retentionDays} days`); +} + +export const handler: Handler = async (event: any, context?: any) => { + const input: ExportTableToPointInTimeInput = { + S3Bucket: DUMP_BUCKET, + TableArn: "arn:aws:dynamodb:us-east-1:153672715141:table/abstract-play-prod", + ExportFormat: "ION", + } + const cmd = new ExportTableToPointInTimeCommand(input); + + try { + const response = await ddbClient.send(cmd); + console.log(`Export command sent:\n${JSON.stringify(response, null, 2)}`) + } catch (err) { + console.log(err) + } + + try { + await pruneOldExports(RETENTION_DAYS); + } catch (err) { + console.error(err); + } + + console.log("ALL DONE"); +}; diff --git a/crons/src/functions/inactive-challenge-cleanup.ts b/crons/src/functions/inactive-challenge-cleanup.ts new file mode 100644 index 00000000..6deb1cea --- /dev/null +++ b/crons/src/functions/inactive-challenge-cleanup.ts @@ -0,0 +1,144 @@ +'use strict'; + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { SESClient } from '@aws-sdk/client-ses'; +import type { Handler } from 'aws-lambda'; +import { + discoverInactiveIssuerChallenges, + isBotId, + issuerStillInactive, +} from '../lib/inactiveChallengeDiscovery.js'; +import { initApbackI18n } from '../lib/apbackI18n.js'; +import { + notifyChallengeRevokedAcceptors, + toRevokeChallengeRecord, +} from '../lib/challengeRevokedNotifications.js'; +import { pauseMatchingRealStandingEntries } from '../lib/pauseRealStanding.js'; +import { revokeChallengeRecord } from '../lib/revokeChallenge.js'; +import type { ChallengeForStandingMatch } from '../lib/standingChallengeMatch.js'; + +const REGION = 'us-east-1'; +const DEFAULT_INACTIVE_MS = 14 * 24 * 60 * 60 * 1000; + +type SkipReason = + | 'issuer_active' + | 'issuer_bot' + | 'invalid_challenge' + | 'error'; + +type Summary = { + inactiveUsers: number; + openChallengesScanned: number; + candidates: number; + revokedStanding: number; + revokedDirect: number; + pausedPresets: number; + skipped: Record; + errors: { challengeId: string; message: string }[]; +}; + +export const handler: Handler = async () => { + const tableName = process.env.ABSTRACT_PLAY_TABLE; + if (tableName === undefined) { + throw new Error('ABSTRACT_PLAY_TABLE is required'); + } + + const inactiveMs = Number(process.env.INACTIVE_CHALLENGE_MS ?? DEFAULT_INACTIVE_MS); + const batchSizeRaw = process.env.INACTIVE_CHALLENGE_REVOKE_BATCH_SIZE; + const batchSize = batchSizeRaw === undefined || batchSizeRaw === '' + ? Number.POSITIVE_INFINITY + : Number(batchSizeRaw); + const inactiveBeforeMs = Date.now() - inactiveMs; + + const ddb = DynamoDBDocumentClient.from(new DynamoDBClient({ region: REGION })); + const ses = new SESClient({ region: REGION }); + await initApbackI18n(); + + const summary: Summary = { + inactiveUsers: 0, + openChallengesScanned: 0, + candidates: 0, + revokedStanding: 0, + revokedDirect: 0, + pausedPresets: 0, + skipped: { + issuer_active: 0, + issuer_bot: 0, + invalid_challenge: 0, + error: 0, + }, + errors: [], + }; + + const discovery = await discoverInactiveIssuerChallenges(ddb, tableName, inactiveBeforeMs); + summary.inactiveUsers = discovery.inactiveUsers; + summary.openChallengesScanned = discovery.openChallengesScanned; + summary.candidates = discovery.candidates.length; + + console.log( + `inactive-challenge-cleanup: ${discovery.candidates.length} candidates ` + + `(${discovery.inactiveUsers} inactive issuers, ${discovery.openChallengesScanned} open challenges scanned)`, + ); + + let processed = 0; + for (const candidate of discovery.candidates) { + if (processed >= batchSize) { + break; + } + try { + if (await isBotId(ddb, tableName, candidate.issuerId)) { + summary.skipped.issuer_bot += 1; + continue; + } + if (!(await issuerStillInactive(ddb, tableName, candidate.issuerId, inactiveBeforeMs))) { + summary.skipped.issuer_active += 1; + continue; + } + + const revokeRecord = toRevokeChallengeRecord(candidate.challenge); + if (revokeRecord === undefined) { + summary.skipped.invalid_challenge += 1; + continue; + } + + const standing = candidate.kind === 'standing'; + await revokeChallengeRecord(ddb, tableName, revokeRecord, standing); + + if (standing) { + summary.pausedPresets += await pauseMatchingRealStandingEntries( + ddb, + tableName, + candidate.issuerId, + candidate.challenge as ChallengeForStandingMatch, + ); + } + + if (standing) { + summary.revokedStanding += 1; + } else { + summary.revokedDirect += 1; + } + processed += 1; + + try { + await notifyChallengeRevokedAcceptors(ddb, tableName, ses, revokeRecord, standing); + } catch (notifyErr) { + console.error( + `Revoked ${candidate.kind} challenge ${candidate.metaGame}#${candidate.id} but notifications failed:`, + notifyErr, + ); + } + } catch (err) { + summary.skipped.error += 1; + summary.errors.push({ + challengeId: candidate.id, + message: err instanceof Error ? err.message : String(err), + }); + console.error(`Failed to revoke ${candidate.kind} challenge ${candidate.metaGame}#${candidate.id}:`, err); + } + } + + console.log('inactive-challenge-cleanup summary:', JSON.stringify(summary)); + return summary; +}; diff --git a/crons/src/functions/player-summary-fanout.ts b/crons/src/functions/player-summary-fanout.ts new file mode 100644 index 00000000..0967a657 --- /dev/null +++ b/crons/src/functions/player-summary-fanout.ts @@ -0,0 +1,101 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import { SQSClient } from "@aws-sdk/client-sqs"; +import type { Handler } from "aws-lambda"; +import { + PLAYER_SUMMARY_MANIFEST_KEY, + SUMMARY_PLAYERS_KEY, + SUMMARY_RATINGS_KEY, + SUMMARY_SITE_KEY, +} from "../constants/recordsBucket.js"; +import { planPlayerSummaryFanout } from "./playerSummaryFanoutPlan.js"; +import { enqueuePlayerSummaryWrites } from "./playerSummaryQueue.js"; +import type { StatSummarySite, StatSummaryPlayers, StatSummaryRatings } from "types/stats/StatSummaryTiers.js"; +import { + parsePreviousPlayerSummaryManifest, + type PlayerSummaryManifest, +} from "types/playerSummaryQueue.js"; +import { getRecordsJson, putRecordsJson, tryGetRecordsJson } from "../utils/recordsJson.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({ region: REGION }); +const sqs = new SQSClient({ region: REGION }); + +export type PlayerSummaryFanoutMetrics = { + generated: string; + candidateCount: number; + enqueuedCount: number; + skippedCount: number; + inputUnchanged: boolean; + tierBytesLoaded: number; + manifestBytes: number; +}; + +export const handler: Handler = async (): Promise => { + const queueUrl = process.env.PLAYER_SUMMARY_QUEUE_URL; + if (queueUrl === undefined || queueUrl === "") { + throw new Error("PLAYER_SUMMARY_QUEUE_URL is not configured"); + } + + const [siteResult, playersResult, ratingsResult, previousManifestResult] = await Promise.all([ + getRecordsJson(s3, SUMMARY_SITE_KEY), + getRecordsJson(s3, SUMMARY_PLAYERS_KEY), + getRecordsJson(s3, SUMMARY_RATINGS_KEY), + tryGetRecordsJson(s3, PLAYER_SUMMARY_MANIFEST_KEY), + ]); + + const tierBytesLoaded = siteResult.bytes + playersResult.bytes + ratingsResult.bytes; + console.log( + `player-summary-fanout: loaded tiers (site=${siteResult.bytes}, ` + + `players=${playersResult.bytes}, ratings=${ratingsResult.bytes} bytes)`, + ); + + const generated = siteResult.data.generated; + const previous = parsePreviousPlayerSummaryManifest(previousManifestResult?.data); + + const plan = planPlayerSummaryFanout({ + generated, + playersTier: playersResult.data, + ratingsTier: ratingsResult.data, + previousHashes: previous.contentHashes, + previousInputFingerprint: previous.inputFingerprint, + }); + + if (plan.inputUnchanged) { + console.log("player-summary-fanout: input unchanged, skipping all enqueues"); + } else if (plan.enqueuedCount > 0) { + console.log(`Enqueueing ${plan.enqueuedCount} player summary writes`); + await enqueuePlayerSummaryWrites(sqs, queueUrl, plan.messages); + } + + console.log( + `player-summary-fanout: candidates=${plan.candidateCount} ` + + `enqueued=${plan.enqueuedCount} skipped=${plan.skippedCount} ` + + `inputUnchanged=${plan.inputUnchanged}`, + ); + + const manifest: PlayerSummaryManifest = { + version: 2, + generated, + enqueuedAt: new Date().toISOString(), + candidateCount: plan.candidateCount, + expectedCount: plan.enqueuedCount, + skippedCount: plan.skippedCount, + inputFingerprint: plan.inputFingerprint, + contentHashes: plan.contentHashes, + }; + const manifestBytes = await putRecordsJson(s3, PLAYER_SUMMARY_MANIFEST_KEY, manifest); + console.log( + `Wrote ${PLAYER_SUMMARY_MANIFEST_KEY} (${manifestBytes} bytes, ` + + `expectedCount=${plan.enqueuedCount}, candidateCount=${plan.candidateCount})`, + ); + + return { + generated, + candidateCount: plan.candidateCount, + enqueuedCount: plan.enqueuedCount, + skippedCount: plan.skippedCount, + inputUnchanged: plan.inputUnchanged, + tierBytesLoaded, + manifestBytes, + }; +}; diff --git a/crons/src/functions/player-summary-worker.ts b/crons/src/functions/player-summary-worker.ts new file mode 100644 index 00000000..2e93bbca --- /dev/null +++ b/crons/src/functions/player-summary-worker.ts @@ -0,0 +1,14 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import type { SQSHandler } from "aws-lambda"; +import type { PlayerSummaryQueueMessage } from "types/playerSummaryQueue.js"; +import { putRecordsJson } from "../utils/recordsJson.js"; + +const s3 = new S3Client({ region: "us-east-1" }); + +export const handler: SQSHandler = async (event) => { + for (const record of event.Records) { + const message = JSON.parse(record.body) as PlayerSummaryQueueMessage; + const bytes = await putRecordsJson(s3, message.key, message.slice); + console.log(`Wrote ${message.key} (${bytes} bytes)`); + } +}; diff --git a/crons/src/functions/playerSummaryFanoutPlan.test.ts b/crons/src/functions/playerSummaryFanoutPlan.test.ts new file mode 100644 index 00000000..e6ad3cc0 --- /dev/null +++ b/crons/src/functions/playerSummaryFanoutPlan.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import type { StatSummary } from "types/stats/StatSummary.js"; +import { + computePlayerSummaryInputFingerprint, + planPlayerSummaryFanout, +} from "./playerSummaryFanoutPlan.js"; +import { + splitStatSummary, + toGlickoStats, + toPlayerSummarySlice, + buildPlayerSummaryIndexesFromTiers, +} from "./summarizeHelpers.js"; +import { playerSummarySliceContentHash } from "../utils/playerSummaryHash.js"; + +const minimalSummary = (): StatSummary => ({ + numGames: 10, + numPlayers: 2, + timeoutRate: 0.1, + abandonedRate: 0.05, + playContext: { casual: 8, event: 2 }, + pieRates: [], + playerCountMix: [], + ratings: { + highest: [ + { user: "a", game: "chess", rating: 1500, wld: [5, 3, 1], glicko: toGlickoStats(1500, 80, 0.06, 9) }, + { user: "b", game: "chess", rating: 1400, wld: [2, 6, 0], glicko: toGlickoStats(1400, 90, 0.06, 8) }, + ], + avg: [{ user: "a", rating: 1500 }, { user: "b", rating: 1400 }], + weighted: [{ user: "a", rating: 1500 }, { user: "b", rating: 1400 }], + glickoByGame: [ + { user: "a", game: "chess", glicko: toGlickoStats(1500, 80, 0.06, 9) }, + { user: "b", game: "chess", glicko: toGlickoStats(1400, 90, 0.06, 8) }, + ], + glickoSite: [], + glickoMeta: { + establishedRd: 110, + provisionalRd: 200, + minGamesEstablished: 20, + minGamesProvisional: 10, + periodMs: 5_184_000_000, + generatedAt: "2026-01-01T00:00:00.000Z", + counts: { byGame: [], site: { rated: 0, provisional: 0, established: 0 } }, + }, + playerCountsByUid: {}, + }, + topPlayers: [], + plays: { total: [], width: [] }, + players: { + allPlays: [{ user: "a", value: 5 }, { user: "b", value: 4 }], + eclectic: [{ user: "a", value: 2 }], + social: [{ user: "a", value: 3 }], + h: [{ user: "a", value: 1 }], + hOpp: [{ user: "b", value: 2 }], + timeoutStats: [{ user: "a", count: 2, latestTimeoutMs: 2_000 }], + }, + histograms: { + all: [1, 2], + allPlayers: [1, 2], + meta: [], + players: [{ user: "a", value: [1, 0] }, { user: "b", value: [0, 1] }], + playerTimeouts: [{ user: "a", value: [1, 1] }, { user: "b", value: [0, 0] }], + firstTimers: [1], + returningPlayers: [0, 1], + activeMovers: [1, 2], + timeouts: [0.1], + abandoned: [0.05], + }, + recent: [], + hoursPer: { mean: 0, median: 0, n: 0, winsorizedCount: 0, byWeek: [] }, + metaStats: {}, + soloMetaStats: {}, + soloSeedBoards: [], + hMeta: [], + geoStats: [], + activeGeoStats: [], + rivalries: [], + pastDisplayNames: [], + seasonality: { + movesByDow: Array.from({ length: 7 }, () => 0), + playersByDow: Array.from({ length: 7 }, () => 0), + movesByHour: Array.from({ length: 24 }, () => 0), + windowDays: 365, + }, +}); + +describe("planPlayerSummaryFanout", () => { + const generated = "2026-01-02T00:00:00.000Z"; + const tiers = () => splitStatSummary(minimalSummary(), generated); + + it("enqueues all candidates when no prior hashes exist", () => { + const { players, ratings } = tiers(); + const plan = planPlayerSummaryFanout({ generated, playersTier: players, ratingsTier: ratings }); + expect(plan.candidateCount).toBe(2); + expect(plan.enqueuedCount).toBe(2); + expect(plan.skippedCount).toBe(0); + expect(plan.messages).toHaveLength(2); + expect(plan.inputUnchanged).toBe(false); + expect(Object.keys(plan.contentHashes)).toEqual(["a", "b"]); + }); + + it("skips users whose slice hash is unchanged", () => { + const { players, ratings } = tiers(); + const first = planPlayerSummaryFanout({ generated, playersTier: players, ratingsTier: ratings }); + const second = planPlayerSummaryFanout({ + generated: "2026-02-02T00:00:00.000Z", + playersTier: players, + ratingsTier: ratings, + previousHashes: first.contentHashes, + previousInputFingerprint: first.inputFingerprint, + }); + expect(second.enqueuedCount).toBe(0); + expect(second.skippedCount).toBe(2); + expect(second.messages).toHaveLength(0); + expect(second.inputUnchanged).toBe(true); + }); + + it("enqueues users whose substantive slice changed", () => { + const { players, ratings } = tiers(); + const first = planPlayerSummaryFanout({ generated, playersTier: players, ratingsTier: ratings }); + const changedPlayers = { + ...players, + players: { + ...players.players, + allPlays: [ + { user: "a", value: 6 }, + { user: "b", value: 4 }, + ], + }, + }; + const second = planPlayerSummaryFanout({ + generated: "2026-02-02T00:00:00.000Z", + playersTier: changedPlayers, + ratingsTier: ratings, + previousHashes: first.contentHashes, + previousInputFingerprint: first.inputFingerprint, + }); + expect(second.inputUnchanged).toBe(false); + expect(second.enqueuedCount).toBe(1); + expect(second.skippedCount).toBe(1); + expect(second.messages).toHaveLength(1); + expect(second.messages[0]!.user).toBe("a"); + }); + + it("input fingerprint ignores tier generated timestamps", () => { + const { players, ratings } = tiers(); + const fingerprintA = computePlayerSummaryInputFingerprint(players, ratings); + const fingerprintB = computePlayerSummaryInputFingerprint( + { ...players, generated: "2099-01-01T00:00:00.000Z" }, + { ...ratings, generated: "2099-01-01T00:00:00.000Z" }, + ); + expect(fingerprintA).toBe(fingerprintB); + }); + + it("content hash ignores generated on slices", () => { + const { players, ratings } = tiers(); + const indexes = buildPlayerSummaryIndexesFromTiers(players, ratings); + const sliceA = toPlayerSummarySlice("a", "2026-01-01T00:00:00.000Z", indexes); + const sliceB = toPlayerSummarySlice("a", "2026-02-02T00:00:00.000Z", indexes); + expect(playerSummarySliceContentHash(sliceA)).toBe(playerSummarySliceContentHash(sliceB)); + }); +}); diff --git a/crons/src/functions/playerSummaryFanoutPlan.ts b/crons/src/functions/playerSummaryFanoutPlan.ts new file mode 100644 index 00000000..5635c669 --- /dev/null +++ b/crons/src/functions/playerSummaryFanoutPlan.ts @@ -0,0 +1,115 @@ +import type { PlayerSummaryQueueMessage } from "types/playerSummaryQueue.js"; +import type { StatSummaryPlayers, StatSummaryRatings } from "types/stats/StatSummaryTiers.js"; +import { playerSummarySliceContentHash, stableJsonHash } from "../utils/playerSummaryHash.js"; +import { + buildPlayerSummaryIndexesFromTiers, + collectPlayerSummaryUserIdsFromTiers, + toPlayerSummarySlice, + type PlayerSummaryIndexes, +} from "./summarizeHelpers.js"; + +export type FanoutPlanInput = { + generated: string; + playersTier: StatSummaryPlayers; + ratingsTier: StatSummaryRatings; + previousHashes?: Record; + previousInputFingerprint?: string; +}; + +export type FanoutPlanResult = { + messages: PlayerSummaryQueueMessage[]; + contentHashes: Record; + candidateCount: number; + enqueuedCount: number; + skippedCount: number; + inputFingerprint: string; + inputUnchanged: boolean; +}; + +function stripTierMeta( + tier: T, +): Omit { + const { generated: _generated, tier: _tier, ...rest } = tier; + return rest; +} + +export function computePlayerSummaryInputFingerprint( + playersTier: StatSummaryPlayers, + ratingsTier: StatSummaryRatings, +): string { + return stableJsonHash({ + players: stripTierMeta(playersTier), + ratings: stripTierMeta(ratingsTier), + }); +} + +function planWithIndexes( + generated: string, + indexes: PlayerSummaryIndexes, + userIds: string[], + previousHashes: Record | undefined, +): Pick { + const messages: PlayerSummaryQueueMessage[] = []; + const contentHashes: Record = {}; + let enqueuedCount = 0; + let skippedCount = 0; + + for (const user of userIds) { + const slice = toPlayerSummarySlice(user, generated, indexes); + const hash = playerSummarySliceContentHash(slice); + contentHashes[user] = hash; + if (previousHashes?.[user] === hash) { + skippedCount += 1; + continue; + } + enqueuedCount += 1; + messages.push({ + user, + key: `player/${user}-summary.json`, + slice, + }); + } + + return { messages, contentHashes, enqueuedCount, skippedCount }; +} + +export function planPlayerSummaryFanout(input: FanoutPlanInput): FanoutPlanResult { + const inputFingerprint = computePlayerSummaryInputFingerprint( + input.playersTier, + input.ratingsTier, + ); + const userIds = collectPlayerSummaryUserIdsFromTiers(input.playersTier, input.ratingsTier); + const candidateCount = userIds.length; + const hasPreviousHashes = input.previousHashes !== undefined + && Object.keys(input.previousHashes).length > 0; + + if ( + hasPreviousHashes + && input.previousInputFingerprint === inputFingerprint + ) { + return { + messages: [], + contentHashes: input.previousHashes!, + candidateCount, + enqueuedCount: 0, + skippedCount: candidateCount, + inputFingerprint, + inputUnchanged: true, + }; + } + + const indexes = buildPlayerSummaryIndexesFromTiers(input.playersTier, input.ratingsTier); + const planned = planWithIndexes( + input.generated, + indexes, + userIds, + input.previousHashes, + ); + + return { + ...planned, + candidateCount, + inputFingerprint, + inputUnchanged: false, + }; +} diff --git a/crons/src/functions/playerSummaryQueue.test.ts b/crons/src/functions/playerSummaryQueue.test.ts new file mode 100644 index 00000000..359cdcb0 --- /dev/null +++ b/crons/src/functions/playerSummaryQueue.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it, vi } from "vitest"; +import { SendMessageBatchCommand } from "@aws-sdk/client-sqs"; +import { enqueuePlayerSummaryWrites } from "./playerSummaryQueue.js"; + +describe("enqueuePlayerSummaryWrites", () => { + it("sends messages in batches of 10", async () => { + const send = vi.fn().mockResolvedValue({ Failed: [] }); + const sqs = { send } as unknown as import("@aws-sdk/client-sqs").SQSClient; + const messages = Array.from({ length: 23 }, (_, i) => ({ + user: `user-${i}`, + key: `player/user-${i}-summary.json`, + slice: { generated: "t", user: `user-${i}`, players: {}, histograms: {}, ratings: { highest: [] } }, + })); + await enqueuePlayerSummaryWrites(sqs, "https://sqs.example/queue", messages); + expect(send).toHaveBeenCalledTimes(3); + expect(send.mock.calls[0]![0]).toBeInstanceOf(SendMessageBatchCommand); + expect(send.mock.calls[0]![0].input.Entries).toHaveLength(10); + expect(send.mock.calls[2]![0].input.Entries).toHaveLength(3); + }); + + it("throws when SQS reports batch failures", async () => { + const send = vi.fn().mockResolvedValue({ + Failed: [{ Id: "0", Code: "InternalError", Message: "boom" }], + }); + const sqs = { send } as unknown as import("@aws-sdk/client-sqs").SQSClient; + await expect(enqueuePlayerSummaryWrites(sqs, "https://sqs.example/queue", [{ + user: "a", + key: "player/a-summary.json", + slice: { generated: "t", user: "a", players: {}, histograms: {}, ratings: { highest: [] } }, + }])).rejects.toThrow(/SendMessageBatch failed/); + }); +}); diff --git a/crons/src/functions/playerSummaryQueue.ts b/crons/src/functions/playerSummaryQueue.ts new file mode 100644 index 00000000..d4f68861 --- /dev/null +++ b/crons/src/functions/playerSummaryQueue.ts @@ -0,0 +1,26 @@ +import { SendMessageBatchCommand, SQSClient } from "@aws-sdk/client-sqs"; +import type { PlayerSummaryQueueMessage } from "types/playerSummaryQueue.js"; + +export const SQS_SEND_BATCH_SIZE = 10; + +export async function enqueuePlayerSummaryWrites( + sqs: SQSClient, + queueUrl: string, + messages: PlayerSummaryQueueMessage[], +): Promise { + for (let i = 0; i < messages.length; i += SQS_SEND_BATCH_SIZE) { + const batch = messages.slice(i, i + SQS_SEND_BATCH_SIZE); + const response = await sqs.send(new SendMessageBatchCommand({ + QueueUrl: queueUrl, + Entries: batch.map((message, index) => ({ + Id: `${i + index}`, + MessageBody: JSON.stringify(message), + })), + })); + if (response.Failed !== undefined && response.Failed.length > 0) { + throw new Error( + `SQS SendMessageBatch failed: ${JSON.stringify(response.Failed)}`, + ); + } + } +} diff --git a/crons/src/functions/rating-change-notifications.ts b/crons/src/functions/rating-change-notifications.ts new file mode 100644 index 00000000..cbe60288 --- /dev/null +++ b/crons/src/functions/rating-change-notifications.ts @@ -0,0 +1,247 @@ +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { + BatchWriteCommand, + BatchGetCommand, + DynamoDBDocumentClient, + QueryCommand, + type QueryCommandInput, +} from "@aws-sdk/lib-dynamodb"; +import { S3Client } from "@aws-sdk/client-s3"; +import type { Handler } from "aws-lambda"; +import { + RATINGS_NOTIFICATION_SNAPSHOT_KEY, + SUMMARY_RATINGS_KEY, +} from "../constants/recordsBucket.js"; +import { + buildRatingChangeSnapshot, + diffRatingChanges, + filterCandidates, + filterCandidatesByInAppPrefs, + ratingChangeConstantsFromEnv, + RATINGS_NOTIFICATION_BASELINE, + toNotificationItems, + type RatingChangeNotificationItem, + type RatingChangeFilterStats, + type RatingNotificationSnapshot, +} from "../lib/ratingChangeNotifications.js"; +import type { InAppNotificationUserSettings } from "../lib/inAppNotificationPrefs.js"; +import type { StatSummaryRatings } from "types/stats/StatSummaryTiers.js"; +import { getRecordsJson, putRecordsJson, tryGetRecordsJson } from "../utils/recordsJson.js"; + +const REGION = "us-east-1"; +const BATCH_WRITE_CHUNK = 25; +const LEGACY_BOT_ID = "SkQfHAjeDxs8eeEnScuYA"; + +const s3 = new S3Client({ region: REGION }); +const ddbClient = new DynamoDBClient({ region: REGION }); +const ddbDocClient = DynamoDBDocumentClient.from(ddbClient, { + marshallOptions: { + convertEmptyValues: false, + removeUndefinedValues: true, + convertClassInstanceToMap: false, + }, +}); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function sendBatchWriteWithRetry( + tableName: string, + items: RatingChangeNotificationItem[], + maxRetries = 8, + initialDelay = 100, + maxDelay = 5000, +): Promise { + for (let offset = 0; offset < items.length; offset += BATCH_WRITE_CHUNK) { + const chunk = items.slice(offset, offset + BATCH_WRITE_CHUNK); + let requestItems = { + [tableName]: chunk.map((item) => ({ PutRequest: { Item: item } })), + }; + let retries = 0; + while (Object.keys(requestItems).length > 0) { + try { + const result = await ddbDocClient.send(new BatchWriteCommand({ + RequestItems: requestItems, + })); + const unprocessed = result.UnprocessedItems; + if (unprocessed === undefined || Object.keys(unprocessed).length === 0) { + break; + } + requestItems = unprocessed; + } catch (err: unknown) { + const name = err instanceof Error ? err.name : ""; + if ( + ["ThrottlingException", "ProvisionedThroughputExceededException", + "InternalServerError", "ServiceUnavailable"].includes(name) + ) { + retries += 1; + if (retries >= maxRetries) { + throw err; + } + const delay = Math.min(initialDelay * Math.pow(2, retries - 1), maxDelay); + const jitter = delay * 0.1 * Math.random(); + await sleep(delay + jitter); + } else { + throw err; + } + } + } + } +} + +async function loadBotIds(tableName: string): Promise> { + const botIds = new Set([LEGACY_BOT_ID]); + const queryInput: QueryCommandInput = { + TableName: tableName, + KeyConditionExpression: "#pk = :pk", + ExpressionAttributeNames: { "#pk": "pk" }, + ExpressionAttributeValues: { ":pk": "BOT" }, + }; + let lastKey: Record | undefined; + do { + const result = await ddbDocClient.send(new QueryCommand({ + ...queryInput, + ExclusiveStartKey: lastKey, + })); + for (const item of result.Items ?? []) { + const sk = item.sk; + if (typeof sk === "string") { + botIds.add(sk); + } + } + lastKey = result.LastEvaluatedKey; + } while (lastKey); + return botIds; +} + +async function loadInAppSettingsForUsers( + tableName: string, + userIds: string[], +): Promise> { + const uniqueIds = [...new Set(userIds)]; + const settings = new Map(); + if (uniqueIds.length === 0) { + return settings; + } + + for (let offset = 0; offset < uniqueIds.length; offset += 100) { + const chunk = uniqueIds.slice(offset, offset + 100); + const result = await ddbDocClient.send(new BatchGetCommand({ + RequestItems: { + [tableName]: { + Keys: chunk.map(sk => ({ pk: "USER", sk })), + ProjectionExpression: "sk, settings", + }, + }, + })); + for (const item of result.Responses?.[tableName] ?? []) { + const userId = item.sk; + if (typeof userId === "string") { + settings.set(userId, item.settings as InAppNotificationUserSettings | undefined); + } + } + } + + return settings; +} + +export type RatingChangeNotificationsMetrics = { + summaryGeneratedAt: string; + notificationsWritten: number; + seededSnapshot: boolean; + skippedAlreadyProcessed: boolean; + filterStats: RatingChangeFilterStats; +}; + +export const handler: Handler = async (): Promise => { + const tableName = process.env.ABSTRACT_PLAY_TABLE; + if (tableName === undefined || tableName === "") { + throw new Error("ABSTRACT_PLAY_TABLE is not configured"); + } + + const ratingsResult = await getRecordsJson(s3, SUMMARY_RATINGS_KEY); + const summary = ratingsResult.data; + const glickoMeta = summary.ratings.glickoMeta; + const summaryGeneratedAt = glickoMeta.generatedAt; + const constants = ratingChangeConstantsFromEnv(glickoMeta); + + const emptyStats: RatingChangeFilterStats = { + skippedNoActivity: 0, + skippedBelowThreshold: 0, + skippedProvisional: 0, + skippedBot: 0, + skippedInAppPrefs: 0, + }; + + const priorSnapshotResult = await tryGetRecordsJson( + s3, + RATINGS_NOTIFICATION_SNAPSHOT_KEY, + ); + + const priorSnapshot = priorSnapshotResult?.data; + const priorBaseline = priorSnapshot?.baseline; + const snapshotUsable = priorSnapshot !== undefined + && priorBaseline === RATINGS_NOTIFICATION_BASELINE; + + if (!snapshotUsable) { + const snapshot = buildRatingChangeSnapshot(summary, summaryGeneratedAt); + const bytes = await putRecordsJson(s3, RATINGS_NOTIFICATION_SNAPSHOT_KEY, snapshot); + const reason = priorSnapshot === undefined ? "no prior snapshot" : "baseline re-seed"; + console.log( + `rating-change-notifications: seeded snapshot (${bytes} bytes, ${reason}), 0 notifications`, + ); + return { + summaryGeneratedAt, + notificationsWritten: 0, + seededSnapshot: true, + skippedAlreadyProcessed: false, + filterStats: emptyStats, + }; + } + + if (priorSnapshot.summaryGeneratedAt === summaryGeneratedAt) { + console.log("rating-change-notifications: summary unchanged, skipping"); + return { + summaryGeneratedAt, + notificationsWritten: 0, + seededSnapshot: false, + skippedAlreadyProcessed: true, + filterStats: emptyStats, + }; + } + + const diffRows = diffRatingChanges(priorSnapshot, summary.ratings.highest); + const botIds = await loadBotIds(tableName); + const { candidates, stats } = filterCandidates(diffRows, botIds, constants); + const userSettings = await loadInAppSettingsForUsers( + tableName, + candidates.map(candidate => candidate.userId), + ); + const { candidates: inAppCandidates, skippedInAppPrefs } = filterCandidatesByInAppPrefs( + candidates, + userSettings, + ); + stats.skippedInAppPrefs = skippedInAppPrefs; + const notificationItems = toNotificationItems(inAppCandidates); + + if (notificationItems.length > 0) { + await sendBatchWriteWithRetry(tableName, notificationItems); + } + + const newSnapshot = buildRatingChangeSnapshot(summary, summaryGeneratedAt); + const snapshotBytes = await putRecordsJson(s3, RATINGS_NOTIFICATION_SNAPSHOT_KEY, newSnapshot); + + console.log( + `rating-change-notifications: wrote ${notificationItems.length} notifications, ` + + `snapshot ${snapshotBytes} bytes; skipped noActivity=${stats.skippedNoActivity} ` + + `belowThreshold=${stats.skippedBelowThreshold} provisional=${stats.skippedProvisional} ` + + `bot=${stats.skippedBot} inAppPrefs=${stats.skippedInAppPrefs}`, + ); + + return { + summaryGeneratedAt, + notificationsWritten: notificationItems.length, + seededSnapshot: false, + skippedAlreadyProcessed: false, + filterStats: stats, + }; +}; diff --git a/crons/src/functions/records-cooccur.ts b/crons/src/functions/records-cooccur.ts new file mode 100644 index 00000000..b6c4192d --- /dev/null +++ b/crons/src/functions/records-cooccur.ts @@ -0,0 +1,169 @@ +'use strict'; + +import { S3Client, GetObjectCommand, ListObjectsV2Command, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { gunzipSync, strFromU8 } from "fflate"; +import { load as loadIon } from "ion-js"; +import { type BasicRec, type GameRec } from "types/index.js"; +import { + buildCooccurArtifact, + DEFAULT_MIN_COOCCURRENCE, + unionCoPlaySet, +} from "../utils/cooccurPmi.js"; +import { putRecordsJson } from "../utils/recordsJson.js"; +import { skipCompletedGameWithoutState, resolveGameMetaGame } from "../utils/completedGameRec.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({ region: REGION }); +const DUMP_BUCKET = "abstractplay-db-dump"; +const REC_BUCKET = "records.abstractplay.com"; +const COOCCUR_KEY = "recommendations/cooccur.json"; + +type UserRec = { + pk: string; + sk: string; + stars?: string[]; +}; + +function pushToSetMap(map: Map>, key: string, value: string): void { + const existing = map.get(key); + if (existing !== undefined) { + existing.add(value); + } else { + map.set(key, new Set([value])); + } +} + +export const handler: Handler = async () => { + const command = new ListObjectsV2Command({ + Bucket: DUMP_BUCKET, + }); + + const allContents: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(command); + if (Contents === undefined) { + throw new Error("Could not list the bucket contents"); + } + allContents.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + command.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + throw err; + } + + const manifests = allContents.filter((c) => c.Key?.includes("manifest-summary.json")); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + if (latest?.Key === undefined) { + throw new Error("No manifest-summary.json found in dump bucket"); + } + const match = latest.Key.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + const uid = match[1]; + const dataFiles = allContents.filter((c) => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz")); + console.log(`Found ${dataFiles.length} data files for export uid ${uid}`); + + const playedByPlayer = new Map>(); + const starredByPlayer = new Map>(); + + for (const file of dataFiles) { + console.log(`Loading ${file.Key}`); + const getCmd = new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + }); + + try { + const response = await s3.send(getCmd); + const bytes = await response.Body?.transformToByteArray(); + if (bytes === undefined) { + throw new Error(`Could not load bytes from file ${file.Key}`); + } + const ion = gunzipSync(bytes); + let sofar = ""; + let ptr = 0; + const chunk = 1_000_000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes("}}\n")) { + const idx = sofar.indexOf("}}\n"); + const line = sofar.substring(0, idx + 2); + sofar = sofar.substring(idx + 3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + continue; + } + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + const rec = json.Item; + if (rec.pk === "GAME" && rec.sk.includes("#1#")) { + if (skipCompletedGameWithoutState(rec)) { + continue; + } + const gdata = rec as GameRec; + const metaGame = resolveGameMetaGame(gdata); + if (!metaGame || !Array.isArray(gdata.players)) { + console.warn( + `Skipping completed GAME without metaGame or players: sk=${gdata.sk}`, + ); + continue; + } + for (const player of gdata.players) { + pushToSetMap(playedByPlayer, player.id, metaGame); + } + } else if (rec.pk === "USER") { + const user = rec as UserRec; + if (Array.isArray(user.stars)) { + for (const meta of user.stars) { + if (typeof meta === "string" && meta.length > 0) { + pushToSetMap(starredByPlayer, user.sk, meta); + } + } + } + } + } catch (err) { + console.log(`An error occurred while loading an ION record: ${line}`); + console.error(err); + } + } + ptr += chunk; + } + } catch (err) { + console.log(`An error occurred while reading data file ${JSON.stringify(file)}`); + console.error(err); + throw err; + } + } + + const playerIds = new Set([...playedByPlayer.keys(), ...starredByPlayer.keys()]); + const playerCoPlaySets: Set[] = []; + for (const playerId of playerIds) { + const played = playedByPlayer.get(playerId) ?? new Set(); + const starred = starredByPlayer.get(playerId) ?? new Set(); + playerCoPlaySets.push(unionCoPlaySet(played, starred)); + } + + console.log( + `Co-occurrence input: ${playerIds.size} players, ` + + `${playedByPlayer.size} with completed games, ${starredByPlayer.size} with stars`, + ); + + const artifact = buildCooccurArtifact(playerCoPlaySets, { + minCooccurrence: DEFAULT_MIN_COOCCURRENCE, + includeStarredBoost: true, + generatedAt: new Date().toISOString(), + }); + + await putRecordsJson(s3, COOCCUR_KEY, artifact); + console.log(`Wrote ${COOCCUR_KEY} (${Object.keys(artifact.games).length} games with PMI neighbors)`); + console.log("ALL DONE"); +}; diff --git a/crons/src/functions/records-manifest.ts b/crons/src/functions/records-manifest.ts new file mode 100644 index 00000000..f1dde947 --- /dev/null +++ b/crons/src/functions/records-manifest.ts @@ -0,0 +1,44 @@ +'use strict'; + +import { S3Client, ListObjectsV2Command, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { REC_BUCKET } from "../constants/recordsBucket.js"; +import { buildRecordsManifest } from "../utils/recordsManifest.js"; +import { putRecordsJson, RECORDS_MANIFEST_CACHE_CONTROL } from "../utils/recordsJson.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({region: REGION}); + +export const handler: Handler = async (event: any, context?: any) => { + const recListCmd = new ListObjectsV2Command({ + Bucket: REC_BUCKET, + }); + + const recList: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(recListCmd); + if (Contents === undefined) { + throw new Error(`Could not list the bucket contents`); + } + recList.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + recListCmd.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + throw err; + } + + const generated = new Date().toISOString(); + const manifest = buildRecordsManifest(recList, generated); + await putRecordsJson(s3, "_manifest.json", manifest, { + cacheControl: RECORDS_MANIFEST_CACHE_CONTROL, + }); + console.log(`Manifest v${manifest.version} generated (${recList.length} objects)`); + + console.log("ALL DONE"); +}; diff --git a/crons/src/functions/records-move-times.ts b/crons/src/functions/records-move-times.ts new file mode 100644 index 00000000..0c063a0f --- /dev/null +++ b/crons/src/functions/records-move-times.ts @@ -0,0 +1,331 @@ +'use strict'; + +import { S3Client, GetObjectCommand, ListObjectsV2Command, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { GameFactory } from '@abstractplay/gameslib'; +import { gunzipSync, strFromU8 } from "fflate"; +import { load as loadIon } from "ion-js"; +import { type BasicRec, type GameRec, type MoveRec } from "types/index.js"; +import { decompressGameState } from "../utils/gameState.js"; +import { skipCompletedGameWithoutState, gameRecHasPlayableState, resolveGameMetaGame } from "../utils/completedGameRec.js"; +import { computeMoveSeasonality, computeWeeklyActiveMovers, MOVE_SEASONALITY_WINDOW_DAYS } from "../utils/moveSeasonality.js"; +import { putRecordsJson } from "../utils/recordsJson.js"; +import type { SeasonalityStats } from "types/stats/SeasonalityStats.js"; +import type { WeeklyActiveMovers } from "../utils/moveSeasonality.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({region: REGION}); +const DUMP_BUCKET = "abstractplay-db-dump"; +const REC_BUCKET = "records.abstractplay.com"; + +type Entry = { + metaGame: string; + score: number; +}; + +type SummaryRec = { + raw1w: Entry[]; + raw1m: Entry[]; + raw6m: Entry[]; + raw1y: Entry[]; + players1w: Entry[]; + players1m: Entry[]; + players6m: Entry[]; + players1y: Entry[]; + playersSum1w: Entry[]; + playersSum1m: Entry[]; + playersSum6m: Entry[]; + playersSum1y: Entry[]; + seasonality: SeasonalityStats; + weeklyActiveMovers: WeeklyActiveMovers; +}; + +export const handler: Handler = async (event: any, context?: any) => { + // scan bucket for data folder + const command = new ListObjectsV2Command({ + Bucket: DUMP_BUCKET, + }); + + const allContents: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(command); + if (Contents === undefined) { + throw new Error(`Could not list the bucket contents`); + } + allContents.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + command.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + } + + // find the latest `manifest-summary.json` file + const manifests = allContents.filter(c => c.Key?.includes("manifest-summary.json")); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + // from there, extract the UID and list of associated data files + const uid = match[1]; + const dataFiles = allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz")); + console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`); + + // load the data from each data file, but only keep the GAME records + const justGames: GameRec[] = []; + for (const file of dataFiles) { + console.log(`Loading ${file.Key}`); + const command = new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + }); + + try { + const response = await s3.send(command); + // The Body object also has 'transformToByteArray' and 'transformToWebStream' methods. + const bytes = await response.Body?.transformToByteArray(); + if (bytes !== undefined) { + const ion = gunzipSync(bytes); + console.log(`Processing ${ion.length} bytes`); + let sofar = ""; + let ptr = 0; + const chunk = 1000000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes("}}\n")) { + const idx = sofar.indexOf("}}\n"); + const line = sofar.substring(0, idx+2); + sofar = sofar.substring(idx+3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + console.log(`Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`) + } else { + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + const rec = json.Item; + if (rec.pk === "GAME") { + if (skipCompletedGameWithoutState(rec)) { + continue; + } + justGames.push(rec as GameRec); + } + } + } catch (err) { + console.log(`An error occurred while loading an ION record: ${line}`); + console.error(err); + } + } + ptr += chunk; + } + } else { + throw new Error(`Could not load bytes from file ${file.Key}`); + } + } catch (err) { + console.log(`An error occured while reading data files. The specific file was ${JSON.stringify(file)}`) + console.error(err); + } + } + console.log(`Found ${justGames.length} GAME records (active and completed)`); + + const cutoff1w = Date.now() - ( 7 * 24 * 60 * 60 * 1000); + const cutoff1m = Date.now() - ( 30 * 24 * 60 * 60 * 1000); + const cutoff6m = Date.now() - (180 * 24 * 60 * 60 * 1000); + const cutoff1y = Date.now() - (365 * 24 * 60 * 60 * 1000); + + const mvTimes1w: MoveRec[] = []; + const mvTimes1m: MoveRec[] = []; + const mvTimes6m: MoveRec[] = []; + const mvTimes1y: MoveRec[] = []; + for (const gdata of justGames) { + if (!gameRecHasPlayableState(gdata)) { + console.warn( + `Skipping GAME without playable state: sk=${gdata.sk} keys=${Object.keys(gdata).join(",")}`, + ); + continue; + } + const metaGame = resolveGameMetaGame(gdata)!; + const g = GameFactory(metaGame, decompressGameState(gdata.state)); + if (g === undefined) { + throw new Error(`Unable to instantiate ${metaGame} game ${gdata.id} (sk=${gdata.sk}):\n${JSON.stringify(gdata.state)}`); + } + // build rec and store + for (let i = 1; i < g.stack.length; i++) { + const time = new Date(g.stack[i]._timestamp).getTime(); + if (time < cutoff1y) { + continue; + } + const pidx = (i - 1) % g.numplayers; + const player = gdata.players[pidx].id; + const rec = { + metaGame, + player, + time, + }; + mvTimes1y.push(rec); + if (time >= cutoff6m) { + mvTimes6m.push(rec); + } + if (time >= cutoff1m) { + mvTimes1m.push(rec); + } + if (time >= cutoff1w) { + mvTimes1w.push(rec); + } + } + } + console.log(`num mv records: ${mvTimes1y.length}`); + + let histogramOriginMs = cutoff1y; + try { + const allResponse = await s3.send(new GetObjectCommand({ + Bucket: REC_BUCKET, + Key: "ALL.json", + })); + const allBody = await allResponse.Body?.transformToString(); + if (allBody !== undefined) { + const allRecs = JSON.parse(allBody) as { header: { "date-end": string } }[]; + if (allRecs.length > 0) { + histogramOriginMs = Math.min( + ...allRecs.map((rec) => new Date(rec.header["date-end"]).getTime()), + ); + } + } + } catch (err) { + console.log(`Could not load ALL.json for weekly mover bucket origin; using 1y cutoff: ${err}`); + } + + // assemble raw scores + const raw1w: Entry[] = []; + const raw1m: Entry[] = []; + const raw6m: Entry[] = []; + const raw1y: Entry[] = []; + + for (const num of [7, 30, 180, 365]) { + const lst: MoveRec[] = num === 7 ? mvTimes1w : num === 30 ? mvTimes1m : num === 180 ? mvTimes6m : mvTimes1y; + const metas = new Set(lst.map(({metaGame}) => metaGame)); + for (const meta of metas) { + const score = lst.filter(({metaGame}) => metaGame === meta).length; + const rec: Entry = { + metaGame: meta, + score, + }; + if (num === 7) { + raw1w.push(rec); + } else if (num === 30) { + raw1m.push(rec); + } else if (num === 180) { + raw6m.push(rec); + } else { + raw1y.push(rec); + } + } + } + + // assemble players scores + const players1w: Entry[] = []; + const players1m: Entry[] = []; + const players6m: Entry[] = []; + const players1y: Entry[] = []; + + for (const num of [7, 30, 180, 365]) { + const lst: MoveRec[] = num === 7 ? mvTimes1w : num === 30 ? mvTimes1m : num === 180 ? mvTimes6m : mvTimes1y; + const metas = new Set(lst.map(({metaGame}) => metaGame)); + for (const meta of metas) { + const recs = lst.filter(({metaGame}) => metaGame === meta); + const minTime = Math.min(...recs.map(({time}) => time)); + const bucketed: {rec: MoveRec, bucket: number}[] = []; + for (const rec of recs) { + const timeSince = (rec.time - minTime); + const bucket = Math.floor(timeSince / (24 * 60 * 60 * 1000)); + bucketed.push({ + rec, + bucket, + }); + } + const maxBucket = Math.max(...bucketed.map(({bucket}) => bucket)); + // players + let score = 0; + for (let i = 0; i <= maxBucket; i++) { + const tranche = bucketed.filter(({bucket}) => bucket === i); + const players = new Set(tranche.map(({rec}) => rec.player)); + score += players.size; + } + + const rec: Entry = { + metaGame: meta, + score, + }; + if (num === 7) { + players1w.push(rec); + } else if (num === 30) { + players1m.push(rec); + } else if (num === 180) { + players6m.push(rec); + } else { + players1y.push(rec); + } + } + } + + // assemble raw player counts per period + const playersSum1w: Entry[] = []; + const playersSum1m: Entry[] = []; + const playersSum6m: Entry[] = []; + const playersSum1y: Entry[] = []; + + for (const num of [7, 30, 180, 365]) { + const lst: MoveRec[] = num === 7 ? mvTimes1w : num === 30 ? mvTimes1m : num === 180 ? mvTimes6m : mvTimes1y; + const metas = new Set(lst.map(({metaGame}) => metaGame)); + for (const meta of metas) { + const recs = lst.filter(({metaGame}) => metaGame === meta); + const uniques = new Set(); + for (const rec of recs) { + uniques.add(rec.player); + } + + const rec: Entry = { + metaGame: meta, + score: uniques.size, + }; + if (num === 7) { + playersSum1w.push(rec); + } else if (num === 30) { + playersSum1m.push(rec); + } else if (num === 180) { + playersSum6m.push(rec); + } else { + playersSum1y.push(rec); + } + } + } + + const final: SummaryRec = { + raw1w, + raw1m, + raw6m, + raw1y, + players1w, + players1m, + players6m, + players1y, + playersSum1w, + playersSum1m, + playersSum6m, + playersSum1y, + seasonality: computeMoveSeasonality(mvTimes1y, MOVE_SEASONALITY_WINDOW_DAYS), + weeklyActiveMovers: computeWeeklyActiveMovers(mvTimes1y, histogramOriginMs), + }; + + // write files to S3 + // response times + await putRecordsJson(s3, "mvtimes.json", final); + console.log("Move times done"); + + console.log("ALL DONE"); +}; diff --git a/crons/src/functions/records-rec-analytics.ts b/crons/src/functions/records-rec-analytics.ts new file mode 100644 index 00000000..ba3a6b1c --- /dev/null +++ b/crons/src/functions/records-rec-analytics.ts @@ -0,0 +1,282 @@ +'use strict'; + +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, ScanCommand } from "@aws-sdk/lib-dynamodb"; +import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { + type AnalyticsSlice, + type AnalyticsState, + type AnalyticsSummary, + type RawDdbItem, + RECOMMENDS_PK_PREFIX, + WATERMARK_OVERLAP_MS, + FIRST_RUN_LOOKBACK_MS, + DAILY_RETENTION_DAYS, + aggregateEvents, + buildAnalyticsSummary, + buildMarkdownReport, + buildRollingSlice, + ingestRawItems, + itemDedupeKey, + mergeSlices, + parseSkEpochMs, + pruneProcessedKeys, + utcDateKey, +} from "../utils/recAnalytics.js"; + +const REGION = "us-east-1"; +const OPS_BUCKET = "private-ops-153672715141-us-east-1-an"; +const ANALYTICS_PREFIX = "recommendations/analytics"; +const STATE_KEY = `${ANALYTICS_PREFIX}/_state.json`; +const SUMMARY_KEY = `${ANALYTICS_PREFIX}/summary.json`; + +const s3 = new S3Client({ region: REGION }); +const ddbClient = new DynamoDBClient({ region: REGION }); +const ddbDocClient = DynamoDBDocumentClient.from(ddbClient, { + marshallOptions: { convertEmptyValues: false, removeUndefinedValues: true }, + unmarshallOptions: { wrapNumbers: false }, +}); + +async function readJsonFromS3(key: string): Promise { + try { + const response = await s3.send(new GetObjectCommand({ + Bucket: OPS_BUCKET, + Key: key, + })); + const body = await response.Body?.transformToString(); + if (body === undefined || body.length === 0) { + return null; + } + return JSON.parse(body) as T; + } catch (err) { + const code = (err as { name?: string }).name; + if (code === "NoSuchKey" || code === "NotFound") { + return null; + } + throw err; + } +} + +async function writeJsonToS3(key: string, value: unknown): Promise { + await s3.send(new PutObjectCommand({ + Bucket: OPS_BUCKET, + Key: key, + Body: JSON.stringify(value, null, 2), + ContentType: "application/json", + })); +} + +async function writeTextToS3(key: string, body: string): Promise { + await s3.send(new PutObjectCommand({ + Bucket: OPS_BUCKET, + Key: key, + Body: body, + ContentType: "text/markdown; charset=utf-8", + })); +} + +async function loadAnalyticsState(nowMs: number): Promise<{ state: AnalyticsState; scanFromMs: number }> { + const existing = await readJsonFromS3(STATE_KEY); + if (existing !== null) { + const scanFromMs = Math.max(0, existing.lastSkWatermarkMs - WATERMARK_OVERLAP_MS); + return { state: existing, scanFromMs }; + } + return { + state: { + lastRunAt: new Date(0).toISOString(), + lastSkWatermarkMs: nowMs - FIRST_RUN_LOOKBACK_MS, + }, + scanFromMs: nowMs - FIRST_RUN_LOOKBACK_MS, + }; +} + +async function scanRecommendationEvents(tableName: string, minSk: string): Promise { + const items: RawDdbItem[] = []; + let lastKey: Record | undefined; + + do { + const response = await ddbDocClient.send(new ScanCommand({ + TableName: tableName, + FilterExpression: "begins_with(pk, :prefix) AND sk >= :minSk", + ExpressionAttributeValues: { + ":prefix": RECOMMENDS_PK_PREFIX, + ":minSk": minSk, + }, + ExclusiveStartKey: lastKey, + })); + if (response.Items !== undefined) { + items.push(...(response.Items as RawDdbItem[])); + } + lastKey = response.LastEvaluatedKey; + } while (lastKey !== undefined); + + return items; +} + +async function listDailySliceKeys(): Promise { + const prefix = `${ANALYTICS_PREFIX}/daily/`; + const keys: string[] = []; + let continuationToken: string | undefined; + + do { + const response = await s3.send(new ListObjectsV2Command({ + Bucket: OPS_BUCKET, + Prefix: prefix, + ContinuationToken: continuationToken, + })); + for (const obj of response.Contents ?? []) { + if (obj.Key !== undefined && obj.Key.endsWith(".json")) { + keys.push(obj.Key); + } + } + continuationToken = response.IsTruncated === true ? response.NextContinuationToken : undefined; + } while (continuationToken !== undefined); + + return keys; +} + +function dailyKeyForDate(date: string): string { + return `${ANALYTICS_PREFIX}/daily/${date}.json`; +} + +function dateFromDailyKey(key: string): string | null { + const match = key.match(/\/daily\/(\d{4}-\d{2}-\d{2})\.json$/); + return match?.[1] ?? null; +} + +async function loadDailySlices(): Promise> { + const keys = await listDailySliceKeys(); + const slices: Array<{ date: string; slice: AnalyticsSlice }> = []; + + for (const key of keys) { + const date = dateFromDailyKey(key); + if (date === null) { + continue; + } + const slice = await readJsonFromS3(key); + if (slice !== null) { + slices.push({ date, slice }); + } + } + + slices.sort((a, b) => a.date.localeCompare(b.date)); + return slices; +} + +function groupEventsByUtcDate( + events: ReturnType["events"], +): Map["events"]> { + const grouped = new Map["events"]>(); + for (const event of events) { + const date = utcDateKey(event.eventTimeMs); + const bucket = grouped.get(date); + if (bucket !== undefined) { + bucket.push(event); + } else { + grouped.set(date, [event]); + } + } + return grouped; +} + +function pruneRetentionDates(dates: string[], asOfDate: string): Set { + const cutoffMs = Date.parse(`${asOfDate}T00:00:00.000Z`) - (DAILY_RETENTION_DAYS - 1) * 86_400_000; + const cutoffDate = utcDateKey(cutoffMs); + return new Set(dates.filter((date) => date >= cutoffDate && date <= asOfDate)); +} + +export const handler: Handler = async () => { + const tableName = process.env.ABSTRACT_PLAY_TABLE; + if (tableName === undefined || tableName.length === 0) { + throw new Error("ABSTRACT_PLAY_TABLE is not set"); + } + + const nowMs = Date.now(); + const generatedAt = new Date(nowMs).toISOString(); + const runDate = utcDateKey(nowMs); + const { state: previousState, scanFromMs } = await loadAnalyticsState(nowMs); + const minSk = String(scanFromMs); + + console.log(`Scanning ${tableName} for ${RECOMMENDS_PK_PREFIX}* with sk >= ${minSk}`); + + const rawItems = await scanRecommendationEvents(tableName, minSk); + console.log(`Scanned ${rawItems.length} recommendation event rows`); + + const processedKeySet = new Set(previousState.processedKeys ?? []); + const newRawItems = rawItems.filter((item) => { + const key = itemDedupeKey(item); + return key !== null && !processedKeySet.has(key); + }); + console.log(`${newRawItems.length} new events after pk#sk dedup (${rawItems.length - newRawItems.length} skipped)`); + + const { events: uniqueEvents, dataQuality: ingestQuality } = ingestRawItems(newRawItems); + + const windowSlice = aggregateEvents(uniqueEvents); + windowSlice.generatedAt = generatedAt; + windowSlice.window = { + start: new Date(scanFromMs).toISOString(), + end: generatedAt, + }; + windowSlice.dataQuality.eventsSkipped += ingestQuality.eventsSkipped; + windowSlice.dataQuality.parseErrors += ingestQuality.parseErrors; + + const eventsByDate = groupEventsByUtcDate(uniqueEvents); + let dailySlices = await loadDailySlices(); + const dailyMap = new Map(dailySlices.map(({ date, slice }) => [date, slice])); + + for (const [date, dateEvents] of eventsByDate) { + const incremental = aggregateEvents(dateEvents); + const existing = dailyMap.get(date); + const mergedDaily = existing !== undefined ? mergeSlices([existing, incremental]) : incremental; + mergedDaily.generatedAt = generatedAt; + dailyMap.set(date, mergedDaily); + await writeJsonToS3(dailyKeyForDate(date), mergedDaily); + console.log(`Wrote daily slice ${date} (${dateEvents.length} events this run)`); + } + + dailySlices = [...dailyMap.entries()] + .map(([date, slice]) => ({ date, slice })) + .sort((a, b) => a.date.localeCompare(b.date)); + + const retainedDates = pruneRetentionDates(dailySlices.map(({ date }) => date), runDate); + dailySlices = dailySlices.filter(({ date }) => retainedDates.has(date)); + + const summaryBody: AnalyticsSummary = buildAnalyticsSummary(windowSlice, dailySlices, runDate); + await writeJsonToS3(SUMMARY_KEY, summaryBody); + + const priorWeekEndMs = Date.parse(`${runDate}T00:00:00.000Z`) - 7 * 86_400_000; + const priorWeekStartMs = priorWeekEndMs - 6 * 86_400_000; + const priorWeekStart = utcDateKey(priorWeekStartMs); + const priorWeekEnd = utcDateKey(priorWeekEndMs); + const priorWeekDaily = dailySlices.filter(({ date }) => date >= priorWeekStart && date <= priorWeekEnd); + const priorWeek = priorWeekDaily.length > 0 + ? buildRollingSlice(priorWeekDaily, priorWeekEnd, 7) + : undefined; + + const report = buildMarkdownReport({ + runDate, + summary: summaryBody, + priorWeek, + }); + await writeTextToS3(`${ANALYTICS_PREFIX}/report/${runDate}.md`, report); + + const maxEventMs = uniqueEvents.reduce((max, event) => Math.max(max, event.eventTimeMs), previousState.lastSkWatermarkMs); + const newProcessedKeys = [ + ...processedKeySet, + ...newRawItems.map((item) => itemDedupeKey(item)).filter((key): key is string => key !== null), + ]; + const pruneBeforeMs = nowMs - DAILY_RETENTION_DAYS * 86_400_000; + const newState: AnalyticsState = { + lastRunAt: generatedAt, + lastSkWatermarkMs: Math.max(previousState.lastSkWatermarkMs, maxEventMs, nowMs), + processedKeys: pruneProcessedKeys(newProcessedKeys, pruneBeforeMs), + }; + await writeJsonToS3(STATE_KEY, newState); + + console.log( + `Wrote ${SUMMARY_KEY}: shows=${summaryBody.totals.shows}, ` + + `clicks=${summaryBody.totals.clicks}, challenges=${summaryBody.totals.challenges}`, + ); + console.log("ALL DONE"); +}; diff --git a/crons/src/functions/records-ttm.ts b/crons/src/functions/records-ttm.ts new file mode 100644 index 00000000..16fc69ca --- /dev/null +++ b/crons/src/functions/records-ttm.ts @@ -0,0 +1,143 @@ +'use strict'; + +import { S3Client, GetObjectCommand, ListObjectsV2Command, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { GameFactory } from '@abstractplay/gameslib'; +import { gunzipSync, strFromU8 } from "fflate"; +import { load as loadIon } from "ion-js"; +import { type BasicRec, type GameRec } from "types/index.js"; +import { decompressGameState } from "../utils/gameState.js"; +import { putRecordsJson } from "../utils/recordsJson.js"; +import { skipCompletedGameWithoutState } from "../utils/completedGameRec.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({region: REGION}); +const DUMP_BUCKET = "abstractplay-db-dump"; +const REC_BUCKET = "records.abstractplay.com"; + +export const handler: Handler = async (event: any, context?: any) => { + // scan bucket for data folder + const command = new ListObjectsV2Command({ + Bucket: DUMP_BUCKET, + }); + + const allContents: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(command); + if (Contents === undefined) { + throw new Error(`Could not list the bucket contents`); + } + allContents.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + command.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + } + + // find the latest `manifest-summary.json` file + const manifests = allContents.filter(c => c.Key?.includes("manifest-summary.json")); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + // from there, extract the UID and list of associated data files + const uid = match[1]; + const dataFiles = allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz")); + console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`); + + // load the data from each data file, but only keep the GAME records + const justGames: GameRec[] = []; + for (const file of dataFiles) { + console.log(`Loading ${file.Key}`); + const command = new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + }); + + try { + const response = await s3.send(command); + // The Body object also has 'transformToByteArray' and 'transformToWebStream' methods. + const bytes = await response.Body?.transformToByteArray(); + if (bytes !== undefined) { + const ion = gunzipSync(bytes); + console.log(`Processing ${ion.length} bytes`); + let sofar = ""; + let ptr = 0; + const chunk = 1000000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes("}}\n")) { + const idx = sofar.indexOf("}}\n"); + const line = sofar.substring(0, idx+2); + sofar = sofar.substring(idx+3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + console.log(`Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`) + } else { + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + const rec = json.Item; + if ( (rec.pk === "GAME") && (rec.sk.includes("#1#")) ) { + if (skipCompletedGameWithoutState(rec)) { + continue; + } + justGames.push(rec as GameRec); + } + } + } catch (err) { + console.log(`An error occurred while loading an ION record: ${line}`); + console.error(err); + } + } + ptr += chunk; + } + } + } catch (err) { + console.log(`An error occured while reading data files. The specific file was ${JSON.stringify(file)}`) + console.error(err); + } + } + console.log(`Found ${justGames.length} completed GAME records`); + + // for each game, generate a game record and categorize it + const pushToMap = (m: Map, key: string, value: any) => { + if (m.has(key)) { + const current = m.get(key)!; + m.set(key, [...current, value]); + } else { + m.set(key, [value]); + } + } + const ttm = new Map(); + for (const gdata of justGames) { + const g = GameFactory(gdata.metaGame, decompressGameState(gdata.state)); + if (g === undefined) { + throw new Error(`Unable to instantiate ${gdata.metaGame} game ${gdata.id} (sk=${gdata.sk}):\n${JSON.stringify(gdata.state)}`); + } + // calculate response rates + const times: number[] = []; + for (let i = 0; i < g.stack.length - 1; i++) { + const t1 = new Date(g.stack[i]._timestamp).getTime(); + const t2 = new Date(g.stack[i+1]._timestamp).getTime(); + times.push(t2 - t1); + } + times.forEach((t, i) => pushToMap(ttm, gdata.players[i % g.numplayers].id, t)); + } + console.log(`ttm: ${ttm.size}`); + + // write files to S3 + // response times + for (const [player, lst] of ttm.entries()) { + await putRecordsJson(s3, `ttm/${player}.json`, lst); + } + console.log("Response times done"); + + console.log("ALL DONE"); +}; diff --git a/crons/src/functions/records.ts b/crons/src/functions/records.ts new file mode 100644 index 00000000..af3ae34c --- /dev/null +++ b/crons/src/functions/records.ts @@ -0,0 +1,260 @@ +'use strict'; + +import { S3Client, GetObjectCommand, ListObjectsV2Command, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { GameFactory, addResource } from '@abstractplay/gameslib'; +import { type APGameRecord } from '@abstractplay/recranks'; +import { gunzipSync, strFromU8 } from "fflate"; +import { load as loadIon } from "ion-js"; +import { type BasicRec, type GameRec, type Tournament, type OrgEvent, type OrgEventGame } from "types/index.js"; +import i18next from "i18next"; +import type { i18n } from "i18next"; +import enApgames from "@abstractplay/gameslib/locales/en/apgames.json"; +import enApresults from "@abstractplay/gameslib/locales/en/apresults.json"; +import { decompressGameState } from "../utils/gameState.js"; +import { encodeRecordGameId } from "../utils/recordGameId.js"; +import { resolveGameVariantUids } from "../utils/resolveGameVariants.js"; +import { findTournamentForGame } from "../utils/recordTournament.js"; +import { gameRecordIsUnrated } from "../utils/recordUnrated.js"; +import { putRecordsJson } from "../utils/recordsJson.js"; +import { skipCompletedGameWithoutState } from "../utils/completedGameRec.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({region: REGION}); +const DUMP_BUCKET = "abstractplay-db-dump"; +const REC_BUCKET = "records.abstractplay.com"; +/** Legacy built-in AI opponent (see node-backend AIAI_USERID). */ +const LEGACY_BOT_ID = "SkQfHAjeDxs8eeEnScuYA"; + +export const handler: Handler = async (event: any, context?: any) => { + const i18nInstance = i18next as unknown as i18n; + await (i18nInstance + .init({ + lng: "en", + fallbackLng: "en", + debug: true, + }) + .then(async function() { + if (!i18nInstance.isInitialized) { + throw new Error(`i18n is not initialized where it should be!`); + } + addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + // scan bucket for data folder + const command = new ListObjectsV2Command({ + Bucket: DUMP_BUCKET, + }); + + const allContents: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(command); + if (Contents === undefined) { + throw new Error(`Could not list the bucket contents`); + } + allContents.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + command.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + } + + // find the latest `manifest-summary.json` file + const manifests = allContents.filter(c => c.Key?.includes("manifest-summary.json")); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + // from there, extract the UID and list of associated data files + const uid = match[1]; + const dataFiles = allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz")); + console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`); + + // load the data from each data file, but only keep the GAME records + const justGames: GameRec[] = []; + const tournaments: Tournament[] = []; + const events: OrgEvent[] = []; + const eventGames: OrgEventGame[] = []; + const registeredBots = new Set([LEGACY_BOT_ID]); + for (const file of dataFiles) { + console.log(`Loading ${file.Key}`); + const command = new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + }); + + try { + const response = await s3.send(command); + // The Body object also has 'transformToByteArray' and 'transformToWebStream' methods. + const bytes = await response.Body?.transformToByteArray(); + if (bytes !== undefined) { + const ion = gunzipSync(bytes); + console.log(`Processing ${ion.length} bytes`); + let sofar = ""; + let ptr = 0; + const chunk = 1000000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes("}}\n")) { + const idx = sofar.indexOf("}}\n"); + const line = sofar.substring(0, idx+2); + sofar = sofar.substring(idx+3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + console.log(`Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`) + } else { + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + const rec = json.Item; + if ( (rec.pk === "GAME") && (rec.sk.includes("#1#")) ) { + if (skipCompletedGameWithoutState(rec)) { + continue; + } + justGames.push(rec as GameRec); + } else if (rec.pk === "TOURNAMENT" || rec.pk === "COMPLETEDTOURNAMENT") { + tournaments.push(rec as Tournament); + } else if (rec.pk === "ORGEVENT") { + events.push(rec as OrgEvent); + } else if (rec.pk === "ORGEVENTGAME") { + eventGames.push(rec as OrgEventGame); + } else if (rec.pk === "BOT") { + registeredBots.add(rec.sk); + } + } + } catch (err) { + console.log(`An error occurred while loading an ION record: ${line}`); + console.error(err); + } + } + ptr += chunk; + } + } else { + throw new Error(`Could not load bytes from ${file.Key}`); + } + } catch (err) { + console.log(`An error occured while reading data files. The specific file was ${JSON.stringify(file)}`) + console.error(err); + } + } + console.log(`Found ${justGames.length} completed GAME records`); + console.log(`Found ${registeredBots.size} registered bots`); + + // for each game, generate a game record and categorize it + const pushToMap = (m: Map, key: string, value: any) => { + if (m.has(key)) { + const current = m.get(key)!; + m.set(key, [...current, value]); + } else { + m.set(key, [value]); + } + } + const allRecs: APGameRecord[] = []; + const metaRecs = new Map(); + const userRecs = new Map(); + const eventRecs = new Map(); + for (const gdata of justGames) { + const g = GameFactory(gdata.metaGame, decompressGameState(gdata.state)); + if (g === undefined) { + throw new Error(`Unable to instantiate ${gdata.metaGame} game ${gdata.id} (sk=${gdata.sk}):\n${JSON.stringify(gdata.state)}`); + } + let event: string|null = null; + let round: string|null = null; + if (gdata.tournament !== undefined) { + const trec = findTournamentForGame(tournaments, gdata.tournament, gdata.metaGame); + if (trec !== undefined) { + event = `Automated Tournament #${trec.number} (${trec.sk})` + round = "1"; + } else { + console.log(`Could not find a matching tournament record for game record "${gdata.sk}".`); + } + } else if (gdata.event !== undefined) { + const erec = events.find(e => e.sk === gdata.event); + const egrec = eventGames.find(eg => eg.sk === [gdata.event, gdata.id].join("#")); + if (erec !== undefined && egrec !== undefined) { + event = erec.name; + round = egrec.round.toString(); + } else { + console.log(`Could not find a matching event records for game record "${gdata.sk}".`) + } + } + const variantUids = resolveGameVariantUids(g.variants, gdata.variants, { + metaGame: gdata.metaGame, + gameId: gdata.id, + }); + if (variantUids.length > 0 && (g.variants?.length ?? 0) === 0) { + g.variants = variantUids; + } + const unrated = gameRecordIsUnrated(gdata.metaGame, variantUids, gdata.rated); + const rec = g.genRecord({ + uid: encodeRecordGameId(gdata.id, gdata.metaGame, variantUids), + players: gdata.players.map(p => ({ + uid: p.id, + name: p.name, + isai: registeredBots.has(p.id) ? true : undefined, + })), + event: event !== null ? event : undefined, + round: round !== null ? round : undefined, + unrated: unrated ? true : undefined, + }); + if (rec === undefined) { + throw new Error(`Unable to create a game report for ${gdata.metaGame} game ${gdata.id}:\n${JSON.stringify(gdata.state)}`); + } + // check for pie + if ( (gdata.pieInvoked !== undefined) && (gdata.pieInvoked) ) { + rec.header.pied = true; + } + // Solo runs archive independently — multiple ALL.json rows per (userid, challenge-seed) are expected. + allRecs.push(rec); + pushToMap(metaRecs, gdata.metaGame, rec); + for (const p of gdata.players) { + pushToMap(userRecs, p.id, rec); + } + if (event !== null) { + let id: string|undefined; + if (gdata.tournament !== undefined) { + id = gdata.tournament; + } else if (gdata.event !== undefined) { + id = gdata.event; + } + if (id !== undefined) { + pushToMap(eventRecs, id, rec); + } + } + } + console.log(`allRecs: ${allRecs.length}, metaRecs: ${[...metaRecs.keys()].length}, userRecs: ${[...userRecs.keys()].length}, eventRecs: ${[...eventRecs.keys()].length}`); + + // // only print the last 10 LoA records to console then quit + // const loa = metaRecs.get("loa")!.slice(-10); + // for (const rec of loa) { + // console.log(JSON.stringify(rec.header)) + // } + + // write files to S3 + await putRecordsJson(s3, "ALL.json", allRecs); + console.log("All records done"); + for (const [meta, recs] of metaRecs.entries()) { + await putRecordsJson(s3, `meta/${meta}.json`, recs); + } + console.log("Meta games done"); + for (const [player, recs] of userRecs.entries()) { + await putRecordsJson(s3, `player/${player}.json`, recs); + } + console.log("Player recs done"); + for (const [eventid, recs] of eventRecs.entries()) { + await putRecordsJson(s3, `event/${eventid}.json`, recs); + } + console.log("Event recs done"); + + console.log("ALL DONE"); + }) + .catch(err => { + throw new Error(`records handler failed:\n${err}`); + })); +}; diff --git a/crons/src/functions/recordsVariantI18n.test.ts b/crons/src/functions/recordsVariantI18n.test.ts new file mode 100644 index 00000000..aec8672b --- /dev/null +++ b/crons/src/functions/recordsVariantI18n.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; +import { GameFactory, addResource } from "@abstractplay/gameslib"; +import enApgames from "@abstractplay/gameslib/locales/en/apgames.json"; +import enApresults from "@abstractplay/gameslib/locales/en/apresults.json"; +import { encodeRecordGameId } from "../utils/recordGameId.js"; +import { gameRecordIsUnrated } from "../utils/recordUnrated.js"; +import { resolveGameVariantUids } from "../utils/resolveGameVariants.js"; + +const INSTANCE_ID = "f47ac10b-58cc-4372-a567-0e02b2c3d479"; + +describe("records variant i18n", () => { + it("resolves variant labels when generating records (same path as records.ts)", () => { + addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + + const g = GameFactory("archimedes", undefined, ["8x10"]); + expect(g).toBeDefined(); + g!.gameover = true; + + const variants = g!.getVariants(); + for (const label of variants) { + expect(label).not.toMatch(/^variants\./); + } + expect(variants).toContain("8x10 board"); + }); + + it("encodes variant UIDs in gameid when generating records (same path as records.ts)", () => { + addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + + const g = GameFactory("archimedes", undefined, ["8x10"]); + expect(g).toBeDefined(); + g!.gameover = true; + + const variantUids = g!.variants ?? []; + expect(variantUids).toContain("8x10"); + + const rec = g!.genRecord({ + uid: encodeRecordGameId(INSTANCE_ID, "archimedes", variantUids), + players: [ + { uid: "alice", name: "Alice" }, + { uid: "bob", name: "Bob" }, + ], + }); + expect(rec).toBeDefined(); + expect(rec!.header.site.gameid).toBe(`${INSTANCE_ID}#archimedes:8x10`); + }); + + it("sets header.unrated when game record is unrated (same path as records.ts)", () => { + addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + + const g = GameFactory("arimaa", undefined, ["free"]); + expect(g).toBeDefined(); + g!.gameover = true; + + const variantUids = g!.variants ?? []; + const unrated = gameRecordIsUnrated("arimaa", variantUids, true); + expect(unrated).toBe(true); + + const rec = g!.genRecord({ + uid: encodeRecordGameId(INSTANCE_ID, "arimaa", variantUids), + players: [ + { uid: "alice", name: "Alice" }, + { uid: "bob", name: "Bob" }, + ], + unrated: unrated ? true : undefined, + }); + expect(rec!.header.unrated).toBe(true); + }); + + it("reconciles record variants when state variants are empty (amazons retroactive bug)", () => { + addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + + const g = GameFactory("amazons", undefined, ["scrambled"]); + expect(g).toBeDefined(); + g!.gameover = true; + const stateWithEmptyVariants = JSON.parse(g!.serialize()) as { variants: string[] }; + stateWithEmptyVariants.variants = []; + const reloaded = GameFactory("amazons", JSON.stringify(stateWithEmptyVariants)); + expect(reloaded).toBeDefined(); + expect(reloaded!.variants).toEqual([]); + + const recordVariants = ["scrambled"]; + const variantUids = resolveGameVariantUids(reloaded!.variants, recordVariants, { + metaGame: "amazons", + gameId: INSTANCE_ID, + }); + expect(variantUids).toEqual(["scrambled"]); + if (variantUids.length > 0 && (reloaded!.variants?.length ?? 0) === 0) { + reloaded!.variants = variantUids; + } + + const rec = reloaded!.genRecord({ + uid: encodeRecordGameId(INSTANCE_ID, "amazons", variantUids), + players: [ + { uid: "alice", name: "Alice" }, + { uid: "bob", name: "Bob" }, + ], + }); + expect(rec).toBeDefined(); + expect(rec!.header.site.gameid).toBe(`${INSTANCE_ID}#amazons:scrambled`); + expect(rec!.header.game.variants).toContain("Scrambled"); + }); +}); diff --git a/crons/src/functions/s3-to-sqs.ts b/crons/src/functions/s3-to-sqs.ts new file mode 100644 index 00000000..3908888f --- /dev/null +++ b/crons/src/functions/s3-to-sqs.ts @@ -0,0 +1,31 @@ +import { SQSClient, SendMessageCommand } from "@aws-sdk/client-sqs"; +import type { S3Handler } from "aws-lambda"; + +const sqs = new SQSClient({}); + +export const handler: S3Handler = async (event) => { + console.log("Received S3 event:", JSON.stringify(event, null, 2)); + + for (const record of event.Records) { + const bucket = record.s3.bucket.name; + const key = decodeURIComponent(record.s3.object.key.replace(/\+/g, " ")); + + const message = { bucket, key }; + + const queueUrl = process.env.SQS_URL; + if (!queueUrl) { + throw new Error("Missing SQS_URL environment variable"); + } + + await sqs.send( + new SendMessageCommand({ + QueueUrl: queueUrl, + MessageBody: JSON.stringify(message), + }), + ); + + console.log( + `Queued job for ${bucket}/${key} → ${process.env.TARGET_QUEUE_ARN ?? "no ARN set"}`, + ); + } +}; diff --git a/crons/src/functions/sqs-to-render.ts b/crons/src/functions/sqs-to-render.ts new file mode 100644 index 00000000..de2fe3ac --- /dev/null +++ b/crons/src/functions/sqs-to-render.ts @@ -0,0 +1,185 @@ +import { + S3Client, + GetObjectCommand, + PutObjectCommand, + DeleteObjectCommand, +} from "@aws-sdk/client-s3"; +import { addPrefix } from "@abstractplay/renderer"; +import type { IRenderOptions, APRenderRep } from "@abstractplay/renderer"; +import type { SQSHandler } from "aws-lambda"; +import { coalesceRenderFrames, type ThumbnailRenderOutput } from "../utils/thumbnailRenderRep.js"; +import { putThumbnailMetric } from "../utils/cloudwatchMetrics.js"; +import { THUMB_BUCKET } from "../utils/thumbnailConfig.js"; +import { Buffer } from "node:buffer"; +import { customAlphabet } from "nanoid"; +import puppeteer, { type Browser } from "puppeteer-core"; +import chromium from "@sparticuz/chromium"; +import { Readable } from "stream"; + +const genPrefix = customAlphabet( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + 5, +); + +const s3 = new S3Client({}); + +const GOOGLE_FONTS_STYLESHEET = + "https://fonts.googleapis.com/css2?family=Cardo:wght@400;700&family=Josefin+Sans:wght@400;600;700&display=swap"; + +const RENDER_PAGE_HTML = ` + + + + + +
+`; + +function rendererScriptUrl(): string { + const url = process.env.RENDERER_CDN_URL; + if (!url) { + throw new Error("Missing RENDERER_CDN_URL environment variable"); + } + return url; +} + +async function streamToString(stream: Readable): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); + stream.on("error", reject); + stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8"))); + }); +} + +let browser: Browser | null = null; + +export const handler: SQSHandler = async (event) => { + console.log("Received SQS event:", JSON.stringify(event, null, 2)); + + if (!browser) { + browser = await puppeteer.launch({ + args: [ + ...chromium.args, + "--disable-dev-shm-usage", + "--disable-gpu", + "--single-process", + "--no-zygote", + "--no-sandbox", + ], + executablePath: await chromium.executablePath(), + headless: true, + }); + } + if (browser === null) { + throw new Error("Unable to instantiate browser."); + } + console.log("Browser initiated."); + + for (const record of event.Records) { + const { bucket, key } = JSON.parse(record.body) as { bucket: string; key: string }; + const [meta] = key.split("."); + + try { + const obj = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + const data = await streamToString(obj.Body as Readable); + let parsed = JSON.parse(data) as ThumbnailRenderOutput | string; + if (typeof parsed === "string") { + parsed = JSON.parse(parsed) as ThumbnailRenderOutput; + } + const aprender = coalesceRenderFrames(parsed); + + const contextLight = { + background: "#fff", + strokes: "#000", + borders: "#000", + labels: "#000", + annotations: "#000", + fill: "#000", + }; + const contextDark = { + background: "#222", + strokes: "#6d6d6d", + borders: "#000", + labels: "#009fbf", + annotations: "#99cccc", + fill: "#e6f2f2", + }; + const contexts = new Map>([ + ["light", contextLight], + ["dark", contextDark], + ]); + + const page = await browser.newPage(); + await page.setViewport({ width: 800, height: 600 }); + const prefix = genPrefix(); + const renderScriptUrl = rendererScriptUrl(); + const failures: string[] = []; + + for (const [name, context] of contexts.entries()) { + await page.setContent(RENDER_PAGE_HTML, { waitUntil: "load" }); + await page.addScriptTag({ url: renderScriptUrl }); + await page.evaluate(() => document.fonts.ready); + await page.evaluate((pfx, colourContext, renderRep) => { + const opts: IRenderOptions = { + prefix: pfx, + divid: "drawing", + colourContext, + contextGlobal: true, + coloursGlobal: false, + }; + (window as unknown as { APRender: { render: (rep: APRenderRep, o: IRenderOptions) => void } }) + .APRender.render(renderRep, opts); + }, prefix, context, aprender); + + const svgString = await page.evaluate(() => { + const svgEl = document.querySelector("svg"); + return svgEl ? svgEl.outerHTML : null; + }); + + if (svgString === null) { + const msg = `No SVG generated for ${meta}-${name}`; + console.error(msg); + failures.push(msg); + continue; + } + + const prefixed = addPrefix(svgString, { prefix } as IRenderOptions); + const safeSvg = prefixed.replace(/ /g, " "); + const cmd = new PutObjectCommand({ + Bucket: THUMB_BUCKET, + Key: `${meta}-${name}.svg`, + Body: safeSvg, + ContentType: "image/svg+xml", + CacheControl: "public, max-age=86400", + }); + const response = await s3.send(cmd); + if (response["$metadata"].httpStatusCode !== 200) { + const msg = `S3 PutObject failed for ${meta}-${name}.svg: ${JSON.stringify(response)}`; + console.error(msg); + failures.push(msg); + continue; + } + console.log(`Rendered SVG written to ${THUMB_BUCKET}/${meta}-${name}.svg`); + } + + await page.close(); + + if (failures.length > 0) { + throw new Error( + `Thumbnail render failed for ${meta} (${failures.length} variant(s)): ${failures.join("; ")}`, + ); + } + + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + console.log(`Deleted prerender file ${bucket}/${key}`); + } catch (err) { + await putThumbnailMetric("RenderFailure", 1, { meta }); + console.error(`Thumbnail render job failed for ${meta} (${bucket}/${key}):`, err); + throw err; + } + } +}; + \ No newline at end of file diff --git a/crons/src/functions/standingchallenges.ts b/crons/src/functions/standingchallenges.ts new file mode 100644 index 00000000..619267a4 --- /dev/null +++ b/crons/src/functions/standingchallenges.ts @@ -0,0 +1,463 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +'use strict'; + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient, PutCommand, GetCommand, UpdateCommand, QueryCommand, QueryCommandInput } from '@aws-sdk/lib-dynamodb'; +// import crypto from 'crypto'; +import { v4 as uuid } from 'uuid'; +import { Handler } from "aws-lambda"; +import { listActiveCurrentGames } from "../lib/activeGamesForUser.js"; +import { adjustShardedCounts } from "../lib/shardedMetaGameCounts.js"; +import { WriteJournal, loadItem } from "../lib/writeJournal.js"; +import { stringArraysEqual } from "../lib/standingChallengeMatch.js"; + +const REGION = "us-east-1"; +const clnt = new DynamoDBClient({ region: REGION }); +const marshallOptions = { + // Whether to automatically convert empty strings, blobs, and sets to `null`. + convertEmptyValues: false, // false, by default. + // Whether to remove undefined values while marshalling. + removeUndefinedValues: true, // false, by default. + // Whether to convert typeof object to map attribute. + convertClassInstanceToMap: false, // false, by default. +}; +const unmarshallOptions = { + // Whether to return numbers as a string instead of converting them to native JavaScript numbers. + wrapNumbers: false, // false, by default. +}; +const translateConfig = { marshallOptions, unmarshallOptions }; +const ddbDocClient = DynamoDBDocumentClient.from(clnt, translateConfig); + +// Types +type UserSettings = { + [k: string]: any; + all?: { + [k: string]: any; + color?: string; + annotate?: boolean; + notifications?: { + gameStart: boolean; + gameEnd: boolean; + challenges: boolean; + yourturn: boolean; + } + } +}; + +type User = { + id: string; + name: string; + time?: number; + settings?: UserSettings; + draw?: string; +} + +type FullUser = { + pk?: string, + sk?: string, + id: string; + name: string; + email: string; + gamesUpdate?: number; + games: Game[]; + challenges_issued?: Set; + challenges_received?: Set; + challenges_accepted?: Set; + challenges_standing?: Set; + admin: boolean | undefined; + language: string; + country: string; + lastSeen?: number; + settings: UserSettings; + ratings?: { + [metaGame: string]: Rating + }; + stars?: string[]; + tags?: TagList[]; + palettes?: Palette[]; + mayPush?: boolean; +} + +type Rating = { + rating: number; + N: number; + wins: number; + draws: number; +} + +type Game = { + pk?: string, + sk?: string, + id : string; + metaGame: string; + players: User[]; + lastMoveTime: number; + clockHard: boolean; + toMove: string | boolean[]; + note?: string; + seen?: number; + winner?: number[]; + numMoves?: number; + gameStarted?: number; + gameEnded?: number; + lastChat?: number; + variants?: string[]; +} + +type TagList = { + meta: string; + tags: string[]; +} + +type Palette = { + name: string; + colours: string[]; +} + +type FullChallenge = { + pk?: string, + sk?: string, + metaGame: string; + numPlayers: number; + standing?: boolean; + duration?: number; + seating: string; + variants: string[]; + challenger: User; + challengees?: User[]; // players who were challenged + players?: User[]; // players that have accepted + clockStart: number; + clockInc: number; + clockMax: number; + clockHard: boolean; + rated: boolean; + noExplore?: boolean; + comment?: string; + dateIssued?: number; +} + +type StandingChallenge = { + id: string; + metaGame: string; + numPlayers: number; + variants?: string[]; + clockStart: number; + clockInc: number; + clockMax: number; + clockHard: boolean; + rated: boolean; + noExplore?: boolean + limit: number; + sensitivity: "meta"|"variants"; + suspended: boolean; +}; + +type StandingChallengeRec = { + pk: "REALSTANDING"; + sk: string; // user's ID + standing: StandingChallenge[]; +}; + +export const handler: Handler = async (event: any, context?: any) => { + let recCount = 0; + let entryCount = 0; + let issued = 0; + let errors = 0; + try { + // get all users' REALSTANDING records + const recs = await getAllRecs(); + recCount = recs.length; + // for each record (user) + for (const rec of recs) { + try { + // get player profile + const userrec = await ddbDocClient.send( + new GetCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { + "pk": "USER", + "sk": rec.sk, + }, + })); + if (userrec.Item === undefined) { + errors++; + console.log(`Could not load user record for ${rec.sk}`); + continue; + } + const user = userrec.Item as FullUser; + // console.log(`Looking at standing entries for ${user.name} (${user.id})`); + // sort entries by meta/variant (meta first) + const entries = rec.standing.sort((a, b) => { + if (a.sensitivity === b.sensitivity) { + return 0; + } else if (a.sensitivity === "meta") { + return -1; + } else { + return 1; + } + }); + // for each challenge + for (const entry of entries) { + // console.log(`Looking at entry:`, entry); + entryCount++; + if (entry.suspended) { continue; } + let totalExisting = 0; + // count number of metagame games and challenges + let metaCount = 0; + const matchingChallenges: string[] = []; + const userStandingChallenges: string[] = Array.from(user.challenges_standing ?? new Set()); + if (userStandingChallenges.length > 0) { + for (const challenge of userStandingChallenges) { + if (challenge.startsWith(entry.metaGame)) { + metaCount++; + matchingChallenges.push(challenge); + } + } + } + const matchingGames: { metaGame: string; variants?: string[] }[] = []; + const activeGames = await listActiveCurrentGames( + ddbDocClient, + process.env.ABSTRACT_PLAY_TABLE!, + rec.sk, + ); + for (const game of activeGames) { + if (game.metaGame === entry.metaGame) { + metaCount++; + matchingGames.push(game); + } + } + let hasMatchingChallenges = false; + // if sensitivity is simply meta, just record the counts + if (entry.sensitivity === "meta") { + totalExisting = metaCount; + hasMatchingChallenges = matchingChallenges.length > 0; + } + // otherwise, check variant combinations + else { + for (const challenge of matchingChallenges) { + const [meta, id] = challenge.split("#"); + const pk = `STANDINGCHALLENGE#${meta}`; + const sk = id; + const challengeRec = await ddbDocClient.send( + new GetCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { + pk, + sk, + }, + })); + if (challengeRec.Item === undefined) { + errors++; + console.log(`Could not load challenge record for ${challenge}`); + continue; + } + const item = challengeRec.Item as FullChallenge; + if (stringArraysEqual(item.variants, entry.variants || [])) { + totalExisting++; + hasMatchingChallenges = true; + break; + } + } + if (!hasMatchingChallenges) { + for (const game of matchingGames) { + if (stringArraysEqual(game.variants || [], entry.variants || [])) { + totalExisting++; + } + } + } + } + + // if there are matching open challenges, don't do anything + if (hasMatchingChallenges) { + continue; + } + + // if count is below limit, issue new challenges + if (totalExisting < entry.limit) { + const challenge: FullChallenge = { + metaGame: entry.metaGame, + numPlayers: entry.numPlayers, + standing: true, + duration: 1, + seating: "random", + variants: entry.variants === undefined ? [] : [...entry.variants], + challenger: { + id: rec.sk, + name: user.name, + }, + players: [{ + id: rec.sk, + name: user.name, + }], + clockStart: entry.clockStart, + clockInc: entry.clockInc, + clockMax: entry.clockMax, + clockHard: entry.clockHard, + rated: entry.rated, + noExplore: entry.noExplore || false, + comment: "Standing Challenge", + dateIssued: Date.now(), + }; + await newStandingChallenge(rec.sk, challenge); + issued++; + } + } + } catch (userErr) { + errors++; + logGetItemError(userErr); + console.log(`Error processing standing challenges for user ${rec.sk}: ${userErr}`); + } + } + } + catch (error) { + logGetItemError(error); + console.log(`An error occurred processing standing challenges from table ${process.env.ABSTRACT_PLAY_TABLE}`); + return; + } + console.log(`Processed standing challenge records for ${recCount} users with a total of ${entryCount} entries. ${issued} challenges were issued and ${errors} errors were encountered.`); +} + +// Handles errors during GetItem execution. Use recommendations in error messages below to +// add error handling specific to your application use-case. +function logGetItemError(err: unknown) { + if (!err) { + console.error('Encountered error object was empty'); + return; + } + if (!(err as { code: any; message: any; }).code) { + console.error(`An exception occurred, investigate and configure retry strategy. Error: ${JSON.stringify(err)}`); + console.error(err); + return; + } + // here are no API specific errors to handle for GetItem, common DynamoDB API errors are handled below + handleCommonErrors(err as { code: any; message: any; }); +} + +function handleCommonErrors(err: { code: any; message: any; }) { + switch (err.code) { + case 'InternalServerError': + console.error(`Internal Server Error, generally safe to retry with exponential back-off. Error: ${err.message}`); + return; + case 'ProvisionedThroughputExceededException': + console.error(`Request rate is too high. If you're using a custom retry strategy make sure to retry with exponential back-off. ` + + `Otherwise consider reducing frequency of requests or increasing provisioned capacity for your table or secondary index. Error: ${err.message}`); + return; + case 'ResourceNotFoundException': + console.error(`One of the tables was not found, verify table exists before retrying. Error: ${err.message}`); + return; + case 'ServiceUnavailable': + console.error(`Had trouble reaching DynamoDB. generally safe to retry with exponential back-off. Error: ${err.message}`); + return; + case 'ThrottlingException': + console.error(`Request denied due to throttling, generally safe to retry with exponential back-off. Error: ${err.message}`); + return; + case 'UnrecognizedClientException': + console.error(`The request signature is incorrect most likely due to an invalid AWS access key ID or secret key, fix before retrying. ` + + `Error: ${err.message}`); + return; + case 'ValidationException': + console.error(`The input fails to satisfy the constraints specified by DynamoDB, ` + + `fix input before retrying. Error: ${err.message}`); + return; + case 'RequestLimitExceeded': + console.error(`Throughput exceeds the current throughput limit for your account, ` + + `increase account level throughput before retrying. Error: ${err.message}`); + return; + default: + console.error(`An exception occurred, investigate and configure retry strategy. Error: ${err.message}`); + return; + } +} + +async function *queryItemsGenerator(queryInput: QueryCommandInput): AsyncGenerator { + let lastEvaluatedKey: Record | undefined + do { + const { Items, LastEvaluatedKey } = await ddbDocClient + .send(new QueryCommand({ ...queryInput, ExclusiveStartKey: lastEvaluatedKey })); + lastEvaluatedKey = LastEvaluatedKey + if (Items !== undefined) { + yield Items + } + } while (lastEvaluatedKey !== undefined) +} + +async function newStandingChallenge(userid: string, challenge: FullChallenge) { + const tableName = process.env.ABSTRACT_PLAY_TABLE!; + const challengeId = uuid(); + const journal = new WriteJournal(); + const challengePk = "STANDINGCHALLENGE#" + challenge.metaGame; + let countAdjusted = false; + + try { + journal.trackCreate(challengePk, challengeId); + await ddbDocClient.send(new PutCommand({ + TableName: tableName, + Item: { + "pk": challengePk, + "sk": challengeId, + "id": challengeId, + "metaGame": challenge.metaGame, + "numPlayers": challenge.numPlayers, + "standing": challenge.standing, + "duration": challenge.duration, + "seating": challenge.seating, + "variants": challenge.variants, + "challenger": challenge.challenger, + "players": [challenge.challenger], + "clockStart": challenge.clockStart, + "clockInc": challenge.clockInc, + "clockMax": challenge.clockMax, + "clockHard": challenge.clockHard, + "rated": challenge.rated, + "noExplore": challenge.noExplore || false, + "comment": challenge.comment || "", + "dateIssued": challenge.dateIssued, + } + })); + + const userBefore = await loadItem(ddbDocClient, tableName, 'USER', userid); + journal.trackReplace(userBefore, 'USER', userid); + await ddbDocClient.send(new UpdateCommand({ + TableName: tableName, + Key: { "pk": "USER", "sk": userid }, + ExpressionAttributeValues: { ":c": new Set([challenge.metaGame + '#' + challengeId]) }, + ExpressionAttributeNames: { "#cs": "challenges_standing" }, + UpdateExpression: "add #cs :c", + })); + + await adjustShardedCounts(ddbDocClient, tableName, challenge.metaGame, { standingchallenges: 1 }); + countAdjusted = true; + console.log("Successfully added challenge" + challengeId); + } catch (err) { + logGetItemError(err); + if (countAdjusted) { + try { + await adjustShardedCounts(ddbDocClient, tableName, challenge.metaGame, { standingchallenges: -1 }); + } catch (countErr) { + logGetItemError(countErr); + console.log(`Failed to roll back standingchallenges count for ${challenge.metaGame}`); + } + } + await journal.rollback(ddbDocClient, tableName, (cmd) => ddbDocClient.send(cmd)); + throw err; + } +} + +const getAllRecs = async (): Promise => { + const result: StandingChallengeRec[] = [] + const queryInput: QueryCommandInput = { + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { + '#pk': 'pk', + }, + ExpressionAttributeValues: { + ':pk': 'REALSTANDING', + }, + TableName: process.env.ABSTRACT_PLAY_TABLE, + } + for await (const page of queryItemsGenerator(queryInput)) { + result.push(...page as StandingChallengeRec[]); + } + return result; +} + diff --git a/crons/src/functions/starttournaments.ts b/crons/src/functions/starttournaments.ts new file mode 100644 index 00000000..d07e37b2 --- /dev/null +++ b/crons/src/functions/starttournaments.ts @@ -0,0 +1,908 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +'use strict'; + +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { DynamoDBDocumentClient, PutCommand, GetCommand, UpdateCommand, DeleteCommand, QueryCommand } from '@aws-sdk/lib-dynamodb'; +// import crypto from 'crypto'; +import { v4 as uuid } from 'uuid'; +import { gameinfo, GameFactory, GameBase, GameBaseSimultaneous, type APGamesInformation } from '@abstractplay/gameslib'; +import { localizedGameName } from '../lib/gameDisplayName.js'; +import { + changeLanguageForPlayer, + initApbackI18n, +} from '../lib/apbackI18n.js'; +import { SESClient, SendEmailCommand } from '@aws-sdk/client-ses'; +import i18n from 'i18next'; +import { Handler } from "aws-lambda"; +import { assignTournamentPlayerRatings } from "../lib/batchRatings.js"; +import { enqueueGameStartNotifications } from "../lib/gameStartNotifications.js"; +import { enqueueTournamentStartNotifications } from "../lib/tournamentStartNotifications.js"; +import type { InAppNotificationUserSettings } from "../lib/inAppNotificationPrefs.js"; +import { + canonicalPlayerPair, + ensureTournamentGameLink, + existingPairKeys, + findExistingGameForPair, + loadExistingTournamentGames, + type ExistingTournamentGame, +} from "../lib/tournamentPairing.js"; +import { tournamentPlaySupported } from "../lib/tournamentGame.js"; +import { + acquireTournamentStartingLock, + loadItem, + releaseTournamentStartingLock, + WriteJournal, +} from "../lib/writeJournal.js"; +import { prepareGameStateForStorage } from "../utils/gameState.js"; +import { loadSummaryRatingsHighest } from "../utils/summaryRatings.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; + +type StartTournamentOptions = { + resume?: boolean; +}; + +type StartTournamentsEvent = { + tournamentId?: string; + resume?: boolean; +}; + +const REGION = "us-east-1"; +const sesClient = new SESClient({ region: REGION }); +const clnt = new DynamoDBClient({ region: REGION }); +const marshallOptions = { + // Whether to automatically convert empty strings, blobs, and sets to `null`. + convertEmptyValues: false, // false, by default. + // Whether to remove undefined values while marshalling. + removeUndefinedValues: true, // false, by default. + // Whether to convert typeof object to map attribute. + convertClassInstanceToMap: false, // false, by default. +}; +const unmarshallOptions = { + // Whether to return numbers as a string instead of converting them to native JavaScript numbers. + wrapNumbers: false, // false, by default. +}; +const translateConfig = { marshallOptions, unmarshallOptions }; +const ddbDocClient = DynamoDBDocumentClient.from(clnt, translateConfig); +const headers = { + 'content-type': 'application/json', + 'Access-Control-Allow-Origin': '*', + "Access-Control-Allow-Headers": "*", + "Access-Control-Allow-Methods": "*" +}; + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +async function sendCommandWithRetry(command: any, maxRetries = 8, initialDelay = 100, maxDelay = 5000) { + let retries = 0; + while (retries < maxRetries) { + try { + // @ts-ignore + return await ddbDocClient.send(command); + } catch (err: any) { + if (['ThrottlingException', 'ProvisionedThroughputExceededException', 'InternalServerError', 'ServiceUnavailable'].includes(err.name)) { + retries++; + if (retries >= maxRetries) { + console.error(`Command failed after ${maxRetries} retries.`); + throw err; + } + const delay = Math.min(initialDelay * Math.pow(2, retries - 1), maxDelay); + const jitter = delay * 0.1 * Math.random(); + console.log(`Retryable error (${err.name}) caught. Retrying in ${Math.round(delay + jitter)}ms...`); + await sleep(delay + jitter); + } else { + throw err; + } + } + } +} + +// Types +export type UserSettings = { + [k: string]: any; + all?: { + [k: string]: any; + color?: string; + annotate?: boolean; + notifications?: { + gameStart: boolean; + gameEnd: boolean; + challenges: boolean; + yourturn: boolean; + tournamentStart: boolean; + tournamentEnd: boolean; + } + } +}; + +export type UserLastSeen = { + id: string; + name: string; + lastSeen?: number; +}; + +export type User = { + id: string; + name: string; + time?: number; + settings?: UserSettings; + draw?: string; +} + +type FullUser = { + pk?: string, + sk?: string, + id: string; + name: string; + email: string; + gamesUpdate?: number; + games: Game[]; + challenges: { + issued: string[]; + received: string[]; + accepted: string[]; + standing: string[]; + } + admin: boolean | undefined; + language: string; + country: string; + lastSeen?: number; + settings: UserSettings; + ratings?: { + [metaGame: string]: Rating + }; + stars?: string[]; + tags?: TagList[]; + palettes?: Palette[]; + mayPush?: boolean; +} + +type Rating = { + rating: number; + N: number; + wins: number; + draws: number; +} + +type Game = { + pk?: string, + sk?: string, + id : string; + metaGame: string; + players: User[]; + lastMoveTime: number; + clockHard: boolean; + toMove: string | boolean[]; + note?: string; + seen?: number; + winner?: number[]; + numMoves?: number; + gameStarted?: number; + gameEnded?: number; + lastChat?: number; + variants?: string[]; +} + +type Division = { + numGames: number; + numCompleted: number; + processed: boolean; + winnerid?: string; + winner?: string; +}; + +type Tournament = { + pk: string; + sk: string; + id: string; + metaGame: string; + variants: string[]; + number: number; + started: boolean; + dateCreated: number; + datePreviousEnded: number; // 0 means either the first tournament or a restart of the series (after it stopped because not enough participants), 3000000000000 means previous tournament still running. + nextid?: string; + dateStarted?: number; + dateEnded?: number; + divisions?: { + [division: number]: Division; + }; + players?: TournamentPlayer[]; // only on archived tournaments + waiting?: boolean; // tournament does not yet have 4 players +}; + +type TournamentPlayer = { + pk: string; + sk: string; + playerid: string; + playername: string; + once?: boolean; + division?: number; + score?: number; + tiebreak?: number; + rating?: number; + timeout?: boolean; +}; + +type TagList = { + meta: string; + tags: string[]; +} + +type Palette = { + name: string; + colours: string[]; +} + +export const handler: Handler = async (event: StartTournamentsEvent) => { + let count = 0; + let newcount = 0; + let cancelledcount = 0; + let waitingcount = 0; + const targetTournamentId = event?.tournamentId; + const resume = event?.resume === true; + const tableName = process.env.ABSTRACT_PLAY_TABLE!; + try { + console.log("Getting TOURNAMENTs"); + const tournamentsData = await ddbDocClient.send( + new QueryCommand({ + TableName: tableName, + KeyConditionExpression: "#pk = :pk", + ExpressionAttributeValues: { ":pk": "TOURNAMENT" }, + ExpressionAttributeNames: { "#pk": "pk" } + })); + let tournaments = tournamentsData.Items as Tournament[]; + if (targetTournamentId !== undefined) { + tournaments = tournaments.filter(t => t.id === targetTournamentId); + if (tournaments.length === 0) { + console.log(`Tournament ${targetTournamentId} not found`); + return; + } + } + console.log("Getting USERS"); + const data = await ddbDocClient.send( + new QueryCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + KeyConditionExpression: "#pk = :pk", + ExpressionAttributeValues: { ":pk": "USERS" }, + ExpressionAttributeNames: { "#pk": "pk", "#name": "name"}, + ProjectionExpression: "sk, #name, lastSeen" + })); + + let users: UserLastSeen[] = []; + if (data.Items) + users = data.Items?.map(u => ({"id": u.sk, "name": u.name, "lastSeen": u.lastSeen})); + const now = Date.now(); + const oneWeek = 1000 * 60 * 60 * 24 * 7; + const twoWeeks = oneWeek * 2; + console.log(`Found ${tournaments.length} tournaments`); + let ratingsHighest: UserGameRating[]; + try { + ratingsHighest = await loadSummaryRatingsHighest(); + } catch (error) { + console.log(`Unable to load summary ratings: ${error}`); + return; + } + for (const tournament of tournaments) { + const scheduleEligible = !tournament.started + && now > tournament.dateCreated + twoWeeks + && (tournament.datePreviousEnded === 0 || now > tournament.datePreviousEnded + oneWeek); + const resumeEligible = resume + && targetTournamentId === tournament.id + && !tournament.started; + if (scheduleEligible || resumeEligible) { + console.log(`Starting tournament ${tournament.id}${resume ? " (resume)" : ""}`); + try { + const status = await startTournament( + users, + tournament, + ratingsHighest, + { resume: resumeEligible }, + ); + if (status === -1) { + cancelledcount++; + } else if (status === 0) { + waitingcount++; + } else if (status === 1) { + newcount++; + } + } catch (error) { + logGetItemError(error); + console.log(`Failed to start tournament ${tournament.id}: ${error}`); + } + } + } + count = tournaments.length; + } + catch (error) { + logGetItemError(error); + console.log(`Tournament start cron failed loading data from table ${tableName}: ${error}`); + return; + } + console.log(`Checked ${count} tournaments, started ${newcount} new tournaments, waiting for ${waitingcount} tournaments and cancelled ${cancelledcount} tournaments`); +} + +async function getPlayersSlowly(playerIDs: string[]) { + const players: FullUser[] = []; + for (const id of playerIDs) { + try { + const playerData = await sendCommandWithRetry( + new GetCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { + "pk": "USER", "sk": id + }, + }) + ) as { Item?: any }; + players.push(playerData.Item as FullUser); + } catch (error) { + logGetItemError(error); + console.log(`Unable to get player ${id} from table ${process.env.ABSTRACT_PLAY_TABLE}`); + } + } + return players; +} + +async function cancelSignupTournament(tournament: Tournament) { + console.log(`Deleting tournament ${tournament.id}`); + await sendCommandWithRetry( + new DeleteCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { + "pk": "TOURNAMENT", + "sk": tournament.id + }, + })); + const sk = tournament.metaGame + "#" + tournament.variants.sort().join("|"); + await sendCommandWithRetry( + new UpdateCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: {"pk": "TOURNAMENTSCOUNTER", "sk": sk}, + ExpressionAttributeValues: { ":t": true }, + ExpressionAttributeNames: {"#o": "over"}, + UpdateExpression: "set #o = :t" + })); +} + +async function startTournament( + users: UserLastSeen[], + tournament: Tournament, + ratingsHighest: UserGameRating[], + options: StartTournamentOptions = {}, +) { + const tableName = process.env.ABSTRACT_PLAY_TABLE!; + const resume = options.resume === true; + // First, get the players + let playersData; + try { + playersData = await ddbDocClient.send( + new QueryCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + ExpressionAttributeValues: { ":pk": "TOURNAMENTPLAYER", ":sk": tournament.id + '#1#' }, + ExpressionAttributeNames: { "#pk": "pk", "#sk": "sk" }, + KeyConditionExpression: "#pk = :pk and begins_with(#sk, :sk)", + }) + ); + } catch (error) { + logGetItemError(error); + console.log(`Unable to get players for tournament ${tournament.id} from table ${process.env.ABSTRACT_PLAY_TABLE}. Error: ${error}`); + return; + } + const players0 = playersData.Items as TournamentPlayer[]; + const remove: TournamentPlayer[] = []; + const players = players0.filter((player, i) => { + // If the player timed out in their last tournament game, and they haven't been seen in 30 days, remove them from the tournament. + // Unless the tournament is in waiting status, then not seen in 30 days is enough to be removed. + if ( + users?.find(u => u.id === player.playerid)?.lastSeen! < Date.now() - 1000 * 60 * 60 * 24 * 30 + && (tournament.waiting === true || player.timeout === true) + ) { + remove.push(player); + if (player.timeout === true) + console.log(`Removing player ${player.playerid} from tournament ${tournament.id} because of timeout`); + else + console.log(`Removing player ${player.playerid} from tournament ${tournament.id} because they haven't been seen in 30 days`); + return false; + } else + return true; + }); + let returnvalue = 0; + if (!tournamentPlaySupported(tournament.metaGame)) { + try { + console.log(`Cancelling tournament ${tournament.id}: ${tournament.metaGame} does not support playercount 2`); + await cancelSignupTournament(tournament); + } + catch (error) { + logGetItemError(error); + console.log(`Unable to delete tournament ${tournament.id} from table ${process.env.ABSTRACT_PLAY_TABLE}`); + return; + } + returnvalue = -1; + } else if (players.length == 0) { + // Cancel tournament. Everyone is gone. + try { + await cancelSignupTournament(tournament); + } + catch (error) { + logGetItemError(error); + console.log(`Unable to delete tournament ${tournament.id} from table ${process.env.ABSTRACT_PLAY_TABLE}`); + return; + } + /* + try { + for (let player of players0) { + work.push(ddbDocClient.send( + new DeleteCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { + "pk": "TOURNAMENTPLAYER", + "sk": player.sk + }, + }))); + } + } + catch (error) { + logGetItemError(error); + console.log(`Unable to delete tournament players from table ${process.env.ABSTRACT_PLAY_TABLE}`); + return; + } + // Send email to players + await initApbackI18n('en'); + for (let player of playersFull) { + await changeLanguageForPlayer(player); + const metaGameName = localizedGameName(tournament.metaGame); + let body = ''; + if (tournament.variants.length === 0) + body = i18n.t("TournamentCancelBody", { "metaGame": metaGameName, "number": tournament.number }); + else + body = i18n.t("TournamentCancelBodyVariants", { "metaGame": metaGameName, "number": tournament.number, "variants": tournament.variants.join(", ") }); + if ( (player.email !== undefined) && (player.email !== null) && (player.email !== "") ) { + const comm = createSendEmailCommand(player.email, player.name, i18n.t("TournamentCancelSubject", { "metaGame": metaGameName }), body); + work.push(sesClient.send(comm)); + } + } + await Promise.all(work); + console.log("Tournament cancelled"); + */ + returnvalue = -1; + } else if (players.length < 4) { + // Not enough players yet + if (tournament.waiting !== true) { + try { + console.log(`Updating tournament ${tournament.id} to waiting`); + await sendCommandWithRetry(new UpdateCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { "pk": "TOURNAMENT", "sk": tournament.id }, + ExpressionAttributeValues: { ":t": true }, + UpdateExpression: "set waiting = :t" + })); + } + catch (error) { + logGetItemError(error); + console.log(`Unable to update tournament ${tournament.id} to waiting`); + return; + } + } + returnvalue = 0; + } else { + const journal = new WriteJournal(); + try { + // enough players, start the tournament! + const clockStart = 72; + const clockInc = 36; + const clockMax = 120; + assignTournamentPlayerRatings( + players, + ratingsHighest, + tournament.metaGame, + tournament.variants ?? [], + ); + players.sort((a, b) => b.rating! - a.rating!); + const playersFull = await getPlayersSlowly(players.map(p => p.playerid)); + const allGamePlayers = players.map(p => {return {id: p.playerid, name: p.playername, time: clockStart * 3600000} as User}); + const playersFull2: FullUser[] = []; + for (const player of players) + playersFull2.push(playersFull.find(p => p.id === player.playerid)!); + + const lockOk = await acquireTournamentStartingLock( + ddbDocClient, + tableName, + tournament.id, + sendCommandWithRetry, + ); + if (!lockOk) { + return 0; + } + // Create divisions + const numDivisions = Math.ceil(players.length / 10.0); // at most 10 players per division + const divisionSizeSmall = Math.floor(players.length / numDivisions); + const numBigDivisions = players.length - divisionSizeSmall * numDivisions; // big divisions have one more player than small divisions! + // Sort players into divisions by rating + players.sort((a, b) => b.rating! - a.rating!); + let existingGames: ExistingTournamentGame[] = []; + if (resume) { + existingGames = await loadExistingTournamentGames(ddbDocClient, tableName, tournament.id); + console.log(`Resume: found ${existingGames.length} existing game(s) for tournament ${tournament.id}`); + } + const pairedKeys = existingPairKeys(existingGames); + const skipDivisionSetup = resume && players.every(p => p.division !== undefined); + if (!skipDivisionSetup) { + let division = 1; + let divisionCount = 0; + for (const player of players) { + player.division = division; + player.sk = tournament.id + "#" + division.toString() + '#' + player.playerid; + console.log(`Adding player ${player.playerid} to tournament ${tournament.id} in division ${division}`); + const prevPlayer = await loadItem(ddbDocClient, tableName, 'TOURNAMENTPLAYER', player.sk); + journal.trackReplace(prevPlayer, 'TOURNAMENTPLAYER', player.sk); + await sendCommandWithRetry(new PutCommand({ + TableName: tableName, + Item: player + })); + if (division > 1) { + const div1Sk = tournament.id + "#1#" + player.playerid; + console.log(`Deleting player ${player.playerid} from tournament ${tournament.id} with division 1 (so they can be put in the right division)`); + const prevDiv1 = await loadItem(ddbDocClient, tableName, 'TOURNAMENTPLAYER', div1Sk); + journal.trackReplace(prevDiv1, 'TOURNAMENTPLAYER', div1Sk); + await sendCommandWithRetry(new DeleteCommand({ + TableName: tableName, + Key: { + "pk": "TOURNAMENTPLAYER", "sk": div1Sk + }, + })); + } + divisionCount++; + if ((division > numBigDivisions && divisionCount === divisionSizeSmall) || (division <= numBigDivisions && divisionCount === divisionSizeSmall + 1)) { + division++; + divisionCount = 0; + } + } + } + // Create games + const now = Date.now(); + let player0 = 0; + const divisions: { [division: number]: {numGames: number, numCompleted: number, processed: boolean} } = {}; + const randomStart = Math.random() < 0.5 ? 0 : 1; + for (let division = 1; division <= numDivisions; division++) { + divisions[division] = {numGames: 0, numCompleted: 0, processed: false}; + for (let i = 0; i < (division <= numBigDivisions ? divisionSizeSmall + 1 : divisionSizeSmall); i++) { + for (let j = i + 1; j < (division <= numBigDivisions ? divisionSizeSmall + 1 : divisionSizeSmall); j++) { + divisions[division].numGames += 1; + const player1 = player0 + i; + const player2 = player0 + j; + const gamePlayers: User[] = []; + if ((i + j + randomStart) % 2 === 1) { + gamePlayers.push(allGamePlayers[player1]); + gamePlayers.push(allGamePlayers[player2]); + } else { + gamePlayers.push(allGamePlayers[player2]); + gamePlayers.push(allGamePlayers[player1]); + } + const pairKey = canonicalPlayerPair(gamePlayers[0]!.id, gamePlayers[1]!.id); + if (resume && pairedKeys.has(pairKey)) { + const existing = findExistingGameForPair(existingGames, pairKey); + if (existing !== undefined) { + console.log(`Resume: linking existing game ${existing.id} for tournament ${tournament.id}`); + const tgSk = tournament.id + "#" + division.toString() + '#' + existing.id; + const linked = await ensureTournamentGameLink( + ddbDocClient, + tableName, + tournament.id, + division, + existing.id, + gamePlayers[0]!.id, + gamePlayers[1]!.id, + ); + if (linked) { + journal.trackCreate('TOURNAMENTGAME', tgSk); + } + continue; + } + } + const gameId = uuid(); + let whoseTurn: string | boolean[] = "0"; + const info = gameinfo.get(tournament.metaGame); + if (info.flags !== undefined && info.flags.includes('simultaneous')) { + whoseTurn = gamePlayers.map(() => true); + } + const variants = tournament.variants; + let engine; + if (info.playercounts.length > 1) + engine = GameFactory(tournament.metaGame, 2, variants); + else + engine = GameFactory(tournament.metaGame, undefined, variants); + if (!engine) + throw new Error(`Unknown metaGame ${tournament.metaGame}`); + const state = engine.serialize(); + const gameSk = tournament.metaGame + "#0#" + gameId; + const tgSk = tournament.id + "#" + division.toString() + '#' + gameId; + journal.trackCreate('GAME', gameSk); + journal.trackCreate('TOURNAMENTGAME', tgSk); + console.log(`Creating game ${gameId} for tournament ${tournament.id} with division ${division}`); + await sendCommandWithRetry(new PutCommand({ + TableName: tableName, + Item: prepareGameStateForStorage({ + "pk": "GAME", + "sk": gameSk, + "id": gameId, + "metaGame": tournament.metaGame, + "numPlayers": 2, + "rated": true, + "players": info.flags !== undefined && info.flags.includes('perspective') ? + gamePlayers.map((p, ind) => {return (ind === 0 ? p : {...p, settings: {"rotate": 180}})}) + : gamePlayers, + "clockStart": clockStart, + "clockInc": clockInc, + "clockMax": clockMax, + "clockHard": true, + "state": state, + "toMove": whoseTurn, + "lastMoveTime": now, + "gameStarted": now, + "variants": engine.variants, + "tournament": tournament.id, + "division": division + }) + })); + await enqueueGameStartNotifications(ddbDocClient, tableName, { + id: gameId, + metaGame: tournament.metaGame, + variants: engine.variants, + players: gamePlayers.map(p => ({ id: p.id, name: p.name })), + }); + const tournamentGame = { + "pk": "TOURNAMENTGAME", + "sk": tgSk, + "id": gameId, + "player1": gamePlayers[0].id, + "player2": gamePlayers[1].id + }; + console.log(`Adding game ${gameId} to TOURNAMENTGAME list`); + await sendCommandWithRetry(new PutCommand({ + TableName: tableName, + Item: tournamentGame + })); + } + } + player0 += division <= numBigDivisions ? divisionSizeSmall + 1 : divisionSizeSmall; + } + const newTournamentid = uuid(); + const tournamentBefore = await loadItem(ddbDocClient, tableName, 'TOURNAMENT', tournament.id); + journal.trackReplace(tournamentBefore, 'TOURNAMENT', tournament.id); + const counterSk = tournament.metaGame + "#" + tournament.variants.sort().join("|"); + const counterBefore = await loadItem(ddbDocClient, tableName, 'TOURNAMENTSCOUNTER', counterSk); + journal.trackReplace(counterBefore, 'TOURNAMENTSCOUNTER', counterSk); + + console.log(`Updating tournament ${tournament.id} to started`); + await sendCommandWithRetry(new UpdateCommand({ + TableName: tableName, + Key: { "pk": "TOURNAMENT", "sk": tournament.id }, + ExpressionAttributeValues: { ":dt": now, ":t": true, ":nextid": newTournamentid, ":ds": divisions }, + UpdateExpression: "set started = :t, dateStarted = :dt, nextid = :nextid, divisions = :ds REMOVE starting, startAttemptAt" + })); + + console.log(`Opening next tournament ${newTournamentid} for sign-up. Update TOURNAMENTSCOUNTER for '${counterSk}'`); + await sendCommandWithRetry(new UpdateCommand({ + TableName: tableName, + Key: { "pk": "TOURNAMENTSCOUNTER", "sk": counterSk }, + ExpressionAttributeValues: { ":inc": 1, ":f": false }, + ExpressionAttributeNames: { "#count": "count", "#over": "over" }, + UpdateExpression: "set #count = #count + :inc, #over = :f" + })); + + const data = { + "pk": "TOURNAMENT", + "sk": newTournamentid, + "id": newTournamentid, + "metaGame": tournament.metaGame, + "variants": tournament.variants, + "number": tournament.number + 1, + "started": false, + "dateCreated": now, + "datePreviousEnded": 3000000000000 + }; + journal.trackCreate('TOURNAMENT', newTournamentid); + console.log(`Creating new tournament ${newTournamentid}`); + await sendCommandWithRetry(new PutCommand({ + TableName: tableName, + Item: data + })); + + for (const player of players) { + let once = false; + if (player.once !== undefined && player.once) { + once = true; + } + if (!once) { + const sk = `${newTournamentid}#1#${player.playerid}`; + const playerdata: TournamentPlayer = { + "pk": "TOURNAMENTPLAYER", + "sk": sk, + "playername": player.playername, + "playerid": player.playerid, + }; + journal.trackCreate('TOURNAMENTPLAYER', sk); + console.log(`Adding player ${player.playerid} to new tournament ${newTournamentid}`); + await sendCommandWithRetry(new PutCommand({ + TableName: tableName, + Item: playerdata + })); + } + } + + // Send e-mails to participants (best-effort; tournament is committed) + await initApbackI18n('en'); + for (const player of playersFull2) { + console.log(`Determining whether to send tournamentStart email to the following player:\n${JSON.stringify(player)}`); + // eslint-disable-next-line no-prototype-builtins + if ( (player.settings?.all?.notifications === undefined) || (!player.settings.all.notifications.hasOwnProperty("tournamentStart")) || (player.settings.all.notifications.tournamentStart) ) { + console.log("Sending email"); + await changeLanguageForPlayer(player); + const metaGameName = localizedGameName(tournament.metaGame); + let body = ''; + if (tournament.variants.length === 0) + body = i18n.t("TournamentStartBody", { "metaGame": metaGameName, "number": tournament.number }); + else + body = i18n.t("TournamentStartBodyVariants", { "metaGame": metaGameName, "number": tournament.number, "variants": tournament.variants.join(", ") }); + if ( (player.email !== undefined) && (player.email !== null) && (player.email !== "") ) { + const comm = createSendEmailCommand(player.email, player.name, i18n.t("TournamentStartSubject", { "metaGame": metaGameName }), body); + try { + await sesClient.send(comm); + } catch (emailErr) { + logGetItemError(emailErr); + console.log(`Failed to send tournament start email to ${player.email}`); + } + } + } + } + try { + const settingsByUserId = new Map( + playersFull2.map(p => [p.id, p.settings as InAppNotificationUserSettings | undefined]), + ); + await enqueueTournamentStartNotifications( + ddbDocClient, + tableName, + { + id: tournament.id, + metaGame: tournament.metaGame, + number: tournament.number, + variants: tournament.variants, + }, + playersFull2.map(p => p.id), + settingsByUserId, + ); + } catch (notifyErr) { + logGetItemError(notifyErr); + console.log(`Failed to enqueue tournament start in-app notifications for ${tournament.id}`); + } + returnvalue = 1; + } catch (error) { + logGetItemError(error); + console.log(`Rolling back tournament start for ${tournament.id}: ${error}`); + if (journal.size > 0) { + await journal.rollback(ddbDocClient, tableName, sendCommandWithRetry); + } + await releaseTournamentStartingLock(ddbDocClient, tableName, tournament.id, sendCommandWithRetry); + return; + } + } + // Delete mia players + if (remove.length > 0) { + for (const player of remove) { + console.log(`Deleting tournament player record for ${player.playerid} from tournament ${tournament.id}`); + await sendCommandWithRetry( + new DeleteCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + Key: { + "pk": "TOURNAMENTPLAYER", "sk": player.sk + }, + }) + ); + } + // Let them know they've been removed + let playersFull: FullUser[] = []; + try { + playersFull = await getPlayersSlowly(remove.map(p => p.playerid)); + } catch (error) { + logGetItemError(error); + console.log(`Unable to get removed players for tournament ${tournament.id} from table ${process.env.ABSTRACT_PLAY_TABLE}. Error: ${error}`); + return; + } + await initApbackI18n('en'); + for (const player of playersFull) { + try { + await changeLanguageForPlayer(player); + const metaGameName = localizedGameName(tournament.metaGame); + let body = ''; + if (tournament.variants.length === 0) + body = i18n.t("TournamentRemoveBody", { "metaGame": metaGameName, "number": tournament.number }); + else + body = i18n.t("TournamentRemoveBodyVariants", { "metaGame": metaGameName, "number": tournament.number, "variants": tournament.variants.join(", ") }); + if ( (player.email !== undefined) && (player.email !== null) && (player.email !== "") ) { + const comm = createSendEmailCommand(player.email, player.name, i18n.t("TournamentRemoveSubject", { "metaGame": metaGameName }), body); + await sesClient.send(comm); + } + } catch (error) { + logGetItemError(error); + console.log(`Failed to send email to player ${player.name}, ${player.email}. Error: ${error}`); + } + } + } + return returnvalue; +} + +export function createSendEmailCommand(toAddress: string, player: any, subject: any, body: string) { + console.log("toAddress", toAddress, "player", player, "body", body); + const fullbody = i18n.t("DearPlayer", { player }) + '\r\n\r\n' + body + "\r\n\r\n" + i18n.t("EmailOut"); + return new SendEmailCommand({ + Destination: { + ToAddresses: [ + toAddress + ], + }, + Message: { + Body: { + Text: { + Charset: "UTF-8", + Data: fullbody + }, + }, + Subject: { + Charset: "UTF-8", + Data: subject + }, + }, + Source: "abstractplay@mail.abstractplay.com" + }); +} + +// Handles errors during GetItem execution. Use recommendations in error messages below to +// add error handling specific to your application use-case. +export function logGetItemError(err: unknown) { + if (!err) { + console.error('Encountered error object was empty'); + return; + } + if (!(err as { code: any; message: any; }).code) { + console.error(`An exception occurred, investigate and configure retry strategy. Error: ${JSON.stringify(err)}`); + console.error(err); + return; + } + // here are no API specific errors to handle for GetItem, common DynamoDB API errors are handled below + handleCommonErrors(err as { code: any; message: any; }); +} + +export function handleCommonErrors(err: { code: any; message: any; }) { + switch (err.code) { + case 'InternalServerError': + console.error(`Internal Server Error, generally safe to retry with exponential back-off. Error: ${err.message}`); + return; + case 'ProvisionedThroughputExceededException': + console.error(`Request rate is too high. If you're using a custom retry strategy make sure to retry with exponential back-off. ` + + `Otherwise consider reducing frequency of requests or increasing provisioned capacity for your table or secondary index. Error: ${err.message}`); + return; + case 'ResourceNotFoundException': + console.error(`One of the tables was not found, verify table exists before retrying. Error: ${err.message}`); + return; + case 'ServiceUnavailable': + console.error(`Had trouble reaching DynamoDB. generally safe to retry with exponential back-off. Error: ${err.message}`); + return; + case 'ThrottlingException': + console.error(`Request denied due to throttling, generally safe to retry with exponential back-off. Error: ${err.message}`); + return; + case 'UnrecognizedClientException': + console.error(`The request signature is incorrect most likely due to an invalid AWS access key ID or secret key, fix before retrying. ` + + `Error: ${err.message}`); + return; + case 'ValidationException': + console.error(`The input fails to satisfy the constraints specified by DynamoDB, ` + + `fix input before retrying. Error: ${err.message}`); + return; + case 'RequestLimitExceeded': + console.error(`Throughput exceeds the current throughput limit for your account, ` + + `increase account level throughput before retrying. Error: ${err.message}`); + return; + default: + console.error(`An exception occurred, investigate and configure retry strategy. Error: ${err.message}`); + return; + } +} diff --git a/crons/src/functions/summarize.ts b/crons/src/functions/summarize.ts new file mode 100644 index 00000000..c18573ff --- /dev/null +++ b/crons/src/functions/summarize.ts @@ -0,0 +1,445 @@ +// tslint:disable: no-console +import { PutObjectCommand, S3Client, GetObjectCommand } from "@aws-sdk/client-s3"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; +import { DynamoDBDocumentClient, QueryCommand } from "@aws-sdk/lib-dynamodb"; +import { ELOBasic, type APGameRecord } from "@abstractplay/recranks"; +import { Handler } from "aws-lambda"; +import { isoToCountryCode } from "../utils/isoToCountryCode.js"; +import { streamJsonArrayFromS3 } from "../utils/streamJsonArray.js"; +import { alignWeeklyActiveMovers } from "../utils/moveSeasonality.js"; +import { putRecordsJson } from "../utils/recordsJson.js"; +import { gameinfo } from "@abstractplay/gameslib"; +import { buildPlayerCountsByUid, compareBatchRatings } from "../lib/batchRatings.js"; +import type { UserRating, StatSummary, RivalriesFull } from "types/index.js"; +import type { UserGameRating } from "types/index.js"; +import type { GeoStats } from "types/index.js"; +import { + GLICKO_PERIOD_MS, + GLICKO_ESTABLISHED_RD, + GLICKO_PROVISIONAL_RD, + GLICKO_MIN_GAMES_ESTABLISHED, + GLICKO_MIN_GAMES_PROVISIONAL, + buildGlickoByGame, + computeGlickoSiteRatings, + computeGlickoGameCounts, + computeGlickoSiteCounts, + computeHoursPerStats, + finalizeRivalryPairs, + publishRivalries, + enrichRivalryPairsWithDisplayNames, + RIVALRY_MIN_GAMES, + RIVALRY_PUBLIC_MIN_GAMES, + splitStatSummary, + type RecordGameIdFallback, +} from "./summarizeHelpers.js"; +import { + buildMetaStatsForGame, + buildHMetaForGame, + listMetaShardKeys, + loadMetaShard, + rateMetaGameVariants, + type RatingListEntry, +} from "./summarizeMeta.js"; +import { + buildPieRates, + buildPlayerCountMix, + buildPlayStats, + buildPlayerStats, + buildSiteHistograms, + buildPastDisplayNamesList, + createSummarizeScanState, + scanRecord, + type GameInfoFlags, +} from "./summarizeScan.js"; +import { + accumulateSoloRecord, + buildSoloMetaStats, + buildSoloSeedBoards, + createSoloSummarizeState, +} from "./summarizeSolo.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({ region: REGION }); +const REC_BUCKET = "records.abstractplay.com"; +const MVTIMES_KEY = "mvtimes.json"; +const OPS_BUCKET = "private-ops-153672715141-us-east-1-an"; +const RIVALRIES_OPS_KEY = "stats/rivalries.json"; +const clnt = new DynamoDBClient({ region: REGION }); +const marshallOptions = { + convertEmptyValues: false, + removeUndefinedValues: true, + convertClassInstanceToMap: false, +}; +const unmarshallOptions = { + wrapNumbers: false, +}; +const translateConfig = { marshallOptions, unmarshallOptions }; +const ddbDocClient = DynamoDBDocumentClient.from(clnt, translateConfig); + +async function putSummaryJson(key: string, body: unknown): Promise { + return putRecordsJson(s3, key, body); +} + +function emptyMoveSeasonality() { + return { + movesByDow: Array.from({ length: 7 }, () => 0), + playersByDow: Array.from({ length: 7 }, () => 0), + movesByHour: Array.from({ length: 24 }, () => 0), + windowDays: 365, + }; +} + +async function loadMvtimes(): Promise<{ + seasonality: ReturnType; + weeklyActiveMovers?: { originMs: number; byWeek: number[] }; +}> { + try { + const response = await s3.send(new GetObjectCommand({ + Bucket: REC_BUCKET, + Key: MVTIMES_KEY, + })); + const str = await response.Body?.transformToString(); + if (str === undefined) { + return { seasonality: emptyMoveSeasonality() }; + } + const parsed = JSON.parse(str) as { + seasonality?: ReturnType; + weeklyActiveMovers?: { originMs: number; byWeek: number[] }; + }; + if (parsed.seasonality === undefined) { + console.log("mvtimes.json has no seasonality field; using empty bins"); + } + return { + seasonality: parsed.seasonality ?? emptyMoveSeasonality(), + weeklyActiveMovers: parsed.weeklyActiveMovers, + }; + } catch (err) { + console.log(`Could not load ${MVTIMES_KEY}: ${err}`); + return { seasonality: emptyMoveSeasonality() }; + } +} + +function buildGameInfoByUid(): Map { + const map = new Map(); + for (const info of gameinfo.values()) { + map.set(info.uid, { + name: info.name, + flags: info.flags, + playercounts: info.playercounts, + }); + } + return map; +} + +export const handler: Handler = async () => { + const gameInfoByUid = buildGameInfoByUid(); + const legacyRecordStats = { legacyGameIds: 0, legacyVariantFallbacks: 0 }; + const recordGameIdFallback: RecordGameIdFallback = { + resolveMetaUidFromDisplayName: (displayName) => { + const found = [...gameinfo.values()].find((i) => i.name === displayName); + return found?.uid; + }, + onLegacyGameId: () => { + legacyRecordStats.legacyGameIds++; + }, + onLegacyVariantFallback: () => { + legacyRecordStats.legacyVariantFallbacks++; + }, + }; + const scanState = createSummarizeScanState(); + const soloState = createSoloSummarizeState(); + + console.log("Streaming all game records from ALL.json"); + try { + const count = await streamJsonArrayFromS3( + s3, + REC_BUCKET, + "ALL.json", + (rec) => { + scanRecord(scanState, rec, gameInfoByUid, recordGameIdFallback); + accumulateSoloRecord(soloState, rec, recordGameIdFallback); + }, + ); + if (count !== scanState.numGames) { + throw new Error(`Stream count mismatch: ${count} vs ${scanState.numGames}`); + } + console.log(`Scanned ${count} records`); + console.log( + `Legacy gameid fallbacks: ${legacyRecordStats.legacyGameIds} records, ` + + `${legacyRecordStats.legacyVariantFallbacks} variant fallbacks`, + ); + } catch (err) { + console.log(`Error occurred streaming ALL.json: ${err}`); + return; + } + + if (scanState.numGames === 0) { + console.log("No records found; skipping summarize"); + return; + } + + const numGames = scanState.numGames; + const numPlayers = scanState.playerIDs.size; + const timeoutRate = scanState.siteEndFailures.length / numGames; + const abandonedRate = scanState.siteAbandonments.length / numGames; + const playContext = { casual: scanState.casualGames, event: scanState.eventGames }; + const earliest = scanState.earliestMs ?? 0; + + const pieRates = buildPieRates(scanState); + const soloMetaStats = buildSoloMetaStats(soloState); + const soloSeedBoards = buildSoloSeedBoards(soloState); + const playerCountMix = buildPlayerCountMix(scanState); + const { numPlays, playWidth } = buildPlayStats(scanState); + const playerStats = buildPlayerStats(scanState); + const histograms = buildSiteHistograms(scanState); + + console.log("Loading meta shards for per-game stats and ratings"); + const metaStats: StatSummary["metaStats"] = {}; + const hMeta: StatSummary["hMeta"] = []; + const ratingList: RatingListEntry[] = []; + const rawList: UserGameRating[] = []; + const rater = new ELOBasic(); + + const metaShardKeys = await listMetaShardKeys(s3, REC_BUCKET); + console.log(`Found ${metaShardKeys.length} meta shards`); + for (const metaUid of metaShardKeys) { + const recs = await loadMetaShard(s3, REC_BUCKET, metaUid); + if (recs.length === 0) { + continue; + } + const hEntry = buildHMetaForGame(recs, metaUid); + if (hEntry !== undefined) { + hMeta.push(hEntry); + } + Object.assign(metaStats, buildMetaStatsForGame(recs, metaUid, recordGameIdFallback)); + rateMetaGameVariants(recs, metaUid, rater, ratingList, rawList, recordGameIdFallback); + } + + const ratedGames = new Set(ratingList.map((r) => r.game)); + const ratedPlayers = new Set(ratingList.map((r) => r.user)); + + console.log("Summarizing ratings"); + const avgRatings: UserRating[] = []; + for (const p of ratedPlayers) { + const ratings = ratingList.filter((r) => r.user === p).map((r) => r.rating.rating); + const sum = ratings.reduce((prev, curr) => prev + curr, 0); + avgRatings.push({ user: p, rating: Math.round(sum / ratings.length) }); + } + const weightedRatings: UserRating[] = []; + for (const p of ratedPlayers) { + const counts = ratingList.filter((r) => r.user === p).map((r) => r.rating.recCount); + const totalRecs = counts.reduce((prev, curr) => prev + curr, 0); + const ratings = ratingList + .filter((r) => r.user === p) + .map((r) => r.rating.rating * (r.rating.recCount / totalRecs)); + const sum = ratings.reduce((prev, curr) => prev + curr, 0); + weightedRatings.push({ user: p, rating: Math.round(sum) }); + } + + const glickoByGame = buildGlickoByGame( + rawList + .filter((row) => row.glicko !== undefined) + .map((row) => ({ user: row.user, game: row.game, glicko: row.glicko! })), + ); + const glickoSite = computeGlickoSiteRatings(glickoByGame); + const glickoMeta = { + establishedRd: GLICKO_ESTABLISHED_RD, + provisionalRd: GLICKO_PROVISIONAL_RD, + minGamesEstablished: GLICKO_MIN_GAMES_ESTABLISHED, + minGamesProvisional: GLICKO_MIN_GAMES_PROVISIONAL, + periodMs: GLICKO_PERIOD_MS, + generatedAt: new Date().toISOString(), + counts: { + byGame: computeGlickoGameCounts(glickoByGame), + site: computeGlickoSiteCounts(glickoSite), + }, + }; + + const topPlayers: UserGameRating[] = []; + for (const g of ratedGames) { + const rows = rawList.filter((r) => r.game === g); + rows.sort(compareBatchRatings); + const top = rows[0]; + if (top !== undefined) { + topPlayers.push(top); + } + } + + const playerCountsByUid = buildPlayerCountsByUid(rawList); + + console.log("Calculating hours per move"); + const hoursPerResult = computeHoursPerStats(scanState.hoursPerGames, earliest); + const { winsorizedCount, ...hoursPer } = hoursPerResult; + console.log( + `hoursPer winsorization: ${winsorizedCount} of ${hoursPer.n} records omitted by winsorization (p2-p98)`, + ); + + let users: Record[] | undefined; + try { + const data = await ddbDocClient.send( + new QueryCommand({ + TableName: process.env.ABSTRACT_PLAY_TABLE, + KeyConditionExpression: "#pk = :pk", + ExpressionAttributeValues: { ":pk": "USERS" }, + ExpressionAttributeNames: { "#pk": "pk", "#name": "name" }, + ProjectionExpression: "sk, country, #name, publicRivalries", + ReturnConsumedCapacity: "INDEXES", + }), + ); + users = data.Items as Record[] | undefined; + if (users === undefined) { + throw new Error("Found no users?"); + } + } catch (err) { + console.log(`An error occurred fetching USERS data: ${err}`); + throw err; + } + + const countryCounts = new Map(); + const userCountry = new Map(); + const userDisplayNames = new Map(); + const publicRivalryUsers = new Set(); + for (const user of users) { + if (typeof user.sk === "string") { + if (typeof user.name === "string" && user.name.length > 0) { + userDisplayNames.set(user.sk, user.name); + } + if (user.publicRivalries === true) { + publicRivalryUsers.add(user.sk); + } + } + const alpha2 = typeof user.country === "string" + ? isoToCountryCode(user.country, "alpha2") + : undefined; + if (alpha2 !== undefined) { + countryCounts.set(alpha2, (countryCounts.get(alpha2) ?? 0) + 1); + if (typeof user.sk === "string") { + userCountry.set(user.sk, alpha2); + } + } + } + const geoStats: GeoStats[] = []; + for (const [alpha2, count] of countryCounts.entries()) { + const name = isoToCountryCode(alpha2, "countryName"); + geoStats.push({ code: alpha2, n: count, name: name || alpha2 }); + } + const activeCountryCounts = new Map(); + for (const uid of scanState.recentCompleterIDs) { + const alpha2 = userCountry.get(uid); + if (alpha2 !== undefined) { + activeCountryCounts.set(alpha2, (activeCountryCounts.get(alpha2) ?? 0) + 1); + } + } + const activeGeoStats: GeoStats[] = []; + for (const [alpha2, count] of activeCountryCounts.entries()) { + const name = isoToCountryCode(alpha2, "countryName"); + activeGeoStats.push({ code: alpha2, n: count, name: name || alpha2 }); + } + activeGeoStats.sort((a, b) => b.n - a.n); + + console.log("Calculating rivalries"); + const identifiedRivalryPairs = finalizeRivalryPairs(scanState.rivalryCounts); + const publicRivalries = publishRivalries( + identifiedRivalryPairs.filter((p) => p.n >= RIVALRY_PUBLIC_MIN_GAMES), + publicRivalryUsers, + userDisplayNames, + ); + const mvtimes = await loadMvtimes(); + const seasonality = mvtimes.seasonality; + const activeMovers = alignWeeklyActiveMovers( + mvtimes.weeklyActiveMovers, + earliest, + histograms.maxBucket, + ); + const rivalriesIdentified: RivalriesFull = { + generated: new Date().toISOString(), + minGames: RIVALRY_MIN_GAMES, + pairs: enrichRivalryPairsWithDisplayNames(identifiedRivalryPairs, userDisplayNames), + }; + + const pastDisplayNames = buildPastDisplayNamesList( + scanState.pastNamesByUser, + userDisplayNames, + ); + + const summary: StatSummary = { + numGames, + numPlayers, + oldestRec: scanState.oldest, + newestRec: scanState.newest, + timeoutRate, + abandonedRate, + playContext, + pieRates, + playerCountMix, + ratings: { + highest: rawList, + avg: avgRatings, + weighted: weightedRatings, + glickoByGame, + glickoSite, + glickoMeta, + playerCountsByUid, + }, + topPlayers, + plays: { + total: numPlays, + width: playWidth, + }, + players: { + allPlays: playerStats.allPlays, + eclectic: playerStats.eclectic, + social: playerStats.social, + h: playerStats.h, + hOpp: playerStats.hOpp, + timeoutStats: histograms.timeoutStats, + }, + histograms: { + all: histograms.histAll, + allPlayers: histograms.histAllPlayers, + activeMovers, + playerTimeouts: histograms.histPlayerTimeouts, + meta: histograms.histMeta, + players: histograms.histPlayers, + firstTimers: histograms.firstTimers, + returningPlayers: histograms.returningPlayers, + timeouts: histograms.histTimeouts, + abandoned: histograms.histAbandoned, + }, + hMeta, + hoursPer, + recent: histograms.recent, + metaStats, + soloMetaStats, + soloSeedBoards, + geoStats, + activeGeoStats, + rivalries: publicRivalries, + seasonality, + pastDisplayNames, + }; + + const opsCmd = new PutObjectCommand({ + Bucket: OPS_BUCKET, + Key: RIVALRIES_OPS_KEY, + Body: JSON.stringify(rivalriesIdentified), + }); + const opsResponse = await s3.send(opsCmd); + if (opsResponse.$metadata.httpStatusCode !== 200) { + console.log(opsResponse); + } + + const generated = new Date().toISOString(); + const monolithBytes = await putSummaryJson("_summary.json", summary); + console.log(`Wrote _summary.json (${monolithBytes} bytes)`); + + const tiers = splitStatSummary(summary, generated); + const siteBytes = await putSummaryJson("_summary-site.json", tiers.site); + console.log(`Wrote _summary-site.json (${siteBytes} bytes)`); + const playersBytes = await putSummaryJson("_summary-players.json", tiers.players); + console.log(`Wrote _summary-players.json (${playersBytes} bytes)`); + const ratingsBytes = await putSummaryJson("_summary-ratings.json", tiers.ratings); + console.log(`Wrote _summary-ratings.json (${ratingsBytes} bytes)`); + + console.log("Analysis complete"); +}; diff --git a/crons/src/functions/summarizeGlicko.test.ts b/crons/src/functions/summarizeGlicko.test.ts new file mode 100644 index 00000000..459ac83e --- /dev/null +++ b/crons/src/functions/summarizeGlicko.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { Glicko2, type APGameRecord, type IGlickoRating } from "@abstractplay/recranks"; +import { + GLICKO_PERIOD_MS, + GLICKO_RATING_START, + GLICKO_RD_START, + computeGlickoNumPeriods, + partitionByGlickoPeriod, +} from "./summarizeHelpers.js"; + +function makeGameRecord(opts: { + gameid: string; + dateEnd: string; + p1Userid: string; + p2Userid: string; + p1Result: number; + p2Result: number; +}): APGameRecord { + return { + header: { + game: { name: "Test" }, + site: { name: "Abstract Play", gameid: opts.gameid }, + "date-start": opts.dateEnd, + "date-end": opts.dateEnd, + "date-generated": opts.dateEnd, + players: [ + { name: "A", userid: opts.p1Userid, result: opts.p1Result }, + { name: "B", userid: opts.p2Userid, result: opts.p2Result }, + ], + }, + moves: [["e4", "e5"]], + } as APGameRecord; +} + +/** Mirrors summarize.ts multi-period Glicko loop for regression testing. */ +function runSummarizeStyleGlicko(recs: APGameRecord[]): Map { + const glicko = new Glicko2({ + minRounds: 0, + ratingStart: GLICKO_RATING_START, + rdStart: GLICKO_RD_START, + }); + const oldestMs = new Date(recs.map((r) => r.header["date-end"]).sort()[0]!).getTime(); + const newestMs = new Date(recs.map((r) => r.header["date-end"]).sort().at(-1)!).getTime(); + const delta = newestMs - oldestMs; + const numPeriods = computeGlickoNumPeriods(delta, GLICKO_PERIOD_MS); + const dated = recs.map((rec) => ({ + rec, + dateEndMs: new Date(rec.header["date-end"]).getTime(), + })); + const buckets = partitionByGlickoPeriod(dated, oldestMs, GLICKO_PERIOD_MS, numPeriods); + let toDate = new Map(); + for (let p = 0; p < numPeriods; p++) { + glicko.knownRatings = new Map(toDate); + const results = glicko.runProcessed(buckets[p]!.map((d) => d.rec)); + toDate = new Map(results.ratings as Map); + } + return toDate; +} + +describe("summarize Glicko multi-period loop", () => { + it("inflates RD for inactive players across an empty period", () => { + const period0Rec = makeGameRecord({ + gameid: "g1", + dateEnd: "2024-01-01T10:00:00Z", + p1Userid: "alice", + p2Userid: "bob", + p1Result: 1, + p2Result: 0, + }); + const period1Rec = makeGameRecord({ + gameid: "g2", + dateEnd: "2024-04-01T10:00:00Z", + p1Userid: "alice", + p2Userid: "carol", + p1Result: 1, + p2Result: 0, + }); + + const afterPeriod0 = runSummarizeStyleGlicko([period0Rec]); + const bobAfterPeriod0 = afterPeriod0.get("Abstract Play|bob")!; + const afterPeriod1 = runSummarizeStyleGlicko([period0Rec, period1Rec]); + const bobAfterPeriod1 = afterPeriod1.get("Abstract Play|bob")!; + + expect(bobAfterPeriod1).toBeDefined(); + expect(bobAfterPeriod1!.rd).toBeGreaterThan(bobAfterPeriod0.rd); + expect(bobAfterPeriod1!.rating).toBe(bobAfterPeriod0.rating); + }); +}); diff --git a/crons/src/functions/summarizeHelpers.test.ts b/crons/src/functions/summarizeHelpers.test.ts new file mode 100644 index 00000000..3701c230 --- /dev/null +++ b/crons/src/functions/summarizeHelpers.test.ts @@ -0,0 +1,751 @@ +import { describe, expect, it, vi } from "vitest"; +import type { APGameRecord } from "@abstractplay/recranks"; +import { + GLICKO_PERIOD_MS, + GLICKO_ESTABLISHED_RD, + GLICKO_PROVISIONAL_RD, + GLICKO_MIN_GAMES_ESTABLISHED, + GLICKO_MIN_GAMES_PROVISIONAL, + buildGlickoByGame, + computeGlickoSiteRatings, + computeGlickoGameCounts, + computeGlickoSiteCounts, + toGlickoStats, + isGlickoProvisional, + isGlickoEstablished, + recordPlayerTimeout, + timeoutStatsFromAccumulator, + buildPlayerTimeoutHistograms, + splitStatSummary, + buildPlayerSummaryIndexes, + buildPlayerSummaryIndexesFromTiers, + collectPlayerSummaryUserIds, + collectPlayerSummaryUserIdsFromTiers, + toPlayerSummarySlice, + statSummaryTierKeys, + STAT_SUMMARY_PARTITIONED_KEYS, + type PlayerTimeoutAccumulator, + computeGlickoNumPeriods, + computeHoursPerStats, + computeReturningPlayersPerWeek, + computeRivalryPairs, + anonymizeRivalries, + publishRivalries, + enrichRivalryPairsWithDisplayNames, + computeTimeoutHistogramRates, + findTimeoutPlayerSeat, + gameSupportsMultiPlayerCount, + gameSupportsPie, + getGlickoPeriodIndex, + maxOf, + medianOf, + percentileOf, + partitionByGlickoPeriod, + recordHasAbandoned, + recordHasTimeout, + recordMoveSlotCount, + recordRoundCount, + recordWasPied, + metaGameFromRecord, + variantUidsFromRecord, + variantComboFromRecord, +} from "./summarizeHelpers.js"; +import type { StatSummary } from "types/stats/StatSummary.js"; + +const emptyHoursPer = () => ({ + mean: 0, + median: 0, + n: 0, + winsorizedCount: 0, + byWeek: [] as number[], +}); + +const minimalStatSummary = (overrides: Partial = {}): StatSummary => ({ + numGames: 10, + numPlayers: 2, + timeoutRate: 0.1, + abandonedRate: 0.05, + playContext: { casual: 8, event: 2 }, + pieRates: [], + playerCountMix: [], + ratings: { + highest: [ + { user: "a", game: "chess", rating: 1500, wld: [5, 3, 1], glicko: toGlickoStats(1500, 80, 0.06, 9) }, + { user: "b", game: "chess", rating: 1400, wld: [2, 6, 0], glicko: toGlickoStats(1400, 90, 0.06, 8) }, + ], + avg: [{ user: "a", rating: 1500 }, { user: "b", rating: 1400 }], + weighted: [{ user: "a", rating: 1500 }, { user: "b", rating: 1400 }], + glickoByGame: [ + { user: "a", game: "chess", glicko: toGlickoStats(1500, 80, 0.06, 9) }, + { user: "b", game: "chess", glicko: toGlickoStats(1400, 90, 0.06, 8) }, + ], + glickoSite: [], + glickoMeta: { + establishedRd: 110, + provisionalRd: 200, + minGamesEstablished: 20, + minGamesProvisional: 10, + periodMs: 5_184_000_000, + generatedAt: "2026-01-01T00:00:00.000Z", + counts: { byGame: [], site: { rated: 0, provisional: 0, established: 0 } }, + }, + playerCountsByUid: {}, + }, + topPlayers: [], + plays: { total: [], width: [] }, + players: { + allPlays: [{ user: "a", value: 5 }, { user: "b", value: 4 }], + eclectic: [{ user: "a", value: 2 }], + social: [{ user: "a", value: 3 }], + h: [{ user: "a", value: 1 }], + hOpp: [{ user: "b", value: 2 }], + timeoutStats: [{ user: "a", count: 2, latestTimeoutMs: 2_000 }], + }, + histograms: { + all: [1, 2], + allPlayers: [1, 2], + meta: [], + players: [{ user: "a", value: [1, 0] }, { user: "b", value: [0, 1] }], + playerTimeouts: [{ user: "a", value: [1, 1] }, { user: "b", value: [0, 0] }], + firstTimers: [1], + returningPlayers: [0, 1], + activeMovers: [1, 2], + timeouts: [0.1], + abandoned: [0.05], + }, + recent: [], + hoursPer: emptyHoursPer(), + metaStats: {}, + soloMetaStats: {}, + soloSeedBoards: [], + hMeta: [], + geoStats: [], + activeGeoStats: [], + rivalries: [], + pastDisplayNames: [], + seasonality: { + movesByDow: Array.from({ length: 7 }, () => 0), + playersByDow: Array.from({ length: 7 }, () => 0), + movesByHour: Array.from({ length: 24 }, () => 0), + windowDays: 365, + }, + ...overrides, +}); + +type Moves = APGameRecord["moves"]; + +describe("recordPlayerTimeout / timeoutStatsFromAccumulator", () => { + it("aggregates count and latest timestamp per user", () => { + const acc = new Map(); + recordPlayerTimeout(acc, "a", 1_000); + recordPlayerTimeout(acc, "a", 2_500); + recordPlayerTimeout(acc, "b", 500); + expect(timeoutStatsFromAccumulator(acc)).toEqual([ + { user: "a", count: 2, latestTimeoutMs: 2_500 }, + { user: "b", count: 1, latestTimeoutMs: 500 }, + ]); + }); +}); + +describe("buildPlayerTimeoutHistograms", () => { + const weekMs = 7 * 24 * 60 * 60 * 1000; + const earliest = 0; + + it("builds weekly buckets per user including zero-filled users", () => { + const acc = new Map(); + recordPlayerTimeout(acc, "a", earliest); + recordPlayerTimeout(acc, "a", earliest + weekMs); + const hist = buildPlayerTimeoutHistograms(acc, ["a", "b"], earliest); + expect(hist).toEqual([ + { user: "a", value: [1, 1] }, + { user: "b", value: [] }, + ]); + }); +}); + +describe("splitStatSummary", () => { + it("partitions monolith keys across tiers without overlap", () => { + const summary = minimalStatSummary(); + const generated = "2026-01-02T00:00:00.000Z"; + const tiers = splitStatSummary(summary, generated); + expect(tiers.site.tier).toBe("site"); + expect(tiers.players.tier).toBe("players"); + expect(tiers.ratings.tier).toBe("ratings"); + expect(tiers.site.generated).toBe(generated); + expect(tiers.players.players.timeoutStats).toEqual(summary.players.timeoutStats); + expect(tiers.ratings.ratings.highest).toEqual(summary.ratings.highest); + const tierKeys = statSummaryTierKeys(tiers.site, tiers.players, tiers.ratings); + for (const key of STAT_SUMMARY_PARTITIONED_KEYS) { + expect(tierKeys.has(key)).toBe(true); + } + expect(tierKeys.size).toBe(STAT_SUMMARY_PARTITIONED_KEYS.length); + }); +}); + +describe("toPlayerSummarySlice", () => { + it("includes pastDisplayNames when present in indexes", () => { + const summary = minimalStatSummary({ + pastDisplayNames: [{ user: "a", names: ["Alice Old"] }], + }); + const indexes = buildPlayerSummaryIndexes(summary); + const slice = toPlayerSummarySlice("a", "2026-01-02T00:00:00.000Z", indexes); + expect(slice.pastDisplayNames).toEqual(["Alice Old"]); + }); + + it("returns only the requested user's rows", () => { + const summary = minimalStatSummary(); + const generated = "2026-01-02T00:00:00.000Z"; + const indexes = buildPlayerSummaryIndexes(summary); + const slice = toPlayerSummarySlice("a", generated, indexes); + expect(slice.user).toBe("a"); + expect(slice.ratings.highest.every((row) => row.user === "a")).toBe(true); + expect(slice.players.timeoutCount).toBe(2); + expect(slice.players.latestTimeoutMs).toBe(2_000); + expect(slice.histograms.players).toEqual([1, 0]); + const other = toPlayerSummarySlice("b", generated, indexes); + expect(other.ratings.highest.every((row) => row.user === "b")).toBe(true); + expect(other.players.timeoutCount).toBeUndefined(); + }); + + it("collects user ids from plays, stats, and ratings", () => { + const users = collectPlayerSummaryUserIds(minimalStatSummary()); + expect(users).toEqual(["a", "b"]); + }); + + it("tier-based helpers match monolith-based helpers", () => { + const summary = minimalStatSummary(); + const generated = "2026-01-02T00:00:00.000Z"; + const tiers = splitStatSummary(summary, generated); + const monolithIndexes = buildPlayerSummaryIndexes(summary); + const tierIndexes = buildPlayerSummaryIndexesFromTiers(tiers.players, tiers.ratings); + expect(collectPlayerSummaryUserIdsFromTiers(tiers.players, tiers.ratings)) + .toEqual(collectPlayerSummaryUserIds(summary)); + for (const user of ["a", "b"]) { + expect(toPlayerSummarySlice(user, generated, tierIndexes)) + .toEqual(toPlayerSummarySlice(user, generated, monolithIndexes)); + } + }); +}); + +describe("toGlickoStats", () => { + it("computes rating bounds and dual provisional/established flags", () => { + const stats = toGlickoStats(1500, 100, 0.06, 25); + expect(stats.ratingLow).toBe(1300); + expect(stats.ratingHigh).toBe(1700); + expect(stats.provisional).toBe(false); + expect(stats.established).toBe(true); + expect(stats.n).toBe(25); + }); + + it("marks low-game-count players provisional", () => { + const stats = toGlickoStats(1500, 80, 0.06, 5); + expect(stats.provisional).toBe(true); + expect(stats.established).toBe(false); + }); + + it("marks high-RD players provisional even with many games", () => { + const stats = toGlickoStats(1500, 250, 0.06, 50); + expect(stats.provisional).toBe(true); + expect(stats.established).toBe(false); + }); +}); + +describe("isGlickoProvisional / isGlickoEstablished", () => { + it("uses AP thresholds from constants", () => { + expect(GLICKO_PROVISIONAL_RD).toBe(200); + expect(GLICKO_ESTABLISHED_RD).toBe(110); + expect(GLICKO_MIN_GAMES_PROVISIONAL).toBe(10); + expect(GLICKO_MIN_GAMES_ESTABLISHED).toBe(20); + expect(isGlickoProvisional(199, 15)).toBe(false); + expect(isGlickoProvisional(201, 15)).toBe(true); + expect(isGlickoEstablished(110, 20)).toBe(true); + expect(isGlickoEstablished(111, 20)).toBe(false); + }); +}); + +describe("computeGlickoSiteRatings", () => { + it("weights composite site rating by games played per meta", () => { + const byGame = buildGlickoByGame([ + { user: "a", game: "chess", glicko: toGlickoStats(1600, 50, 0.06, 10) }, + { user: "a", game: "go", glicko: toGlickoStats(1400, 100, 0.06, 30) }, + ]); + const site = computeGlickoSiteRatings(byGame); + expect(site).toHaveLength(1); + const entry = site[0]!; + expect(entry.n).toBe(40); + expect(entry.ratingLow).toBeCloseTo((1500 * 10 + 1200 * 30) / 40); + expect(entry.provisional).toBe(false); + expect(entry.established).toBe(true); + }); +}); + +describe("computeGlickoGameCounts / computeGlickoSiteCounts", () => { + it("counts provisional and established rows per game and site", () => { + const byGame = buildGlickoByGame([ + { user: "a", game: "chess", glicko: toGlickoStats(1500, 80, 0.06, 25) }, + { user: "b", game: "chess", glicko: toGlickoStats(1500, 250, 0.06, 5) }, + { user: "a", game: "go", glicko: toGlickoStats(1500, 80, 0.06, 3) }, + ]); + expect(computeGlickoGameCounts(byGame)).toEqual([ + { game: "chess", rated: 2, provisional: 1, established: 1 }, + { game: "go", rated: 1, provisional: 1, established: 0 }, + ]); + const site = computeGlickoSiteRatings(byGame); + expect(computeGlickoSiteCounts(site)).toEqual({ + rated: 2, + provisional: 2, + established: 1, + }); + }); +}); + +describe("maxOf", () => { + it("returns -1 for an empty array", () => { + expect(maxOf([])).toBe(-1); + }); + + it("returns the maximum value", () => { + expect(maxOf([3, 1])).toBe(3); + }); +}); + +describe("getGlickoPeriodIndex", () => { + const oldest = 1_000_000; + + it("assigns all records to period 0 when numPeriods is 1", () => { + expect(getGlickoPeriodIndex(oldest, oldest, GLICKO_PERIOD_MS, 1)).toBe(0); + expect(getGlickoPeriodIndex(oldest + GLICKO_PERIOD_MS, oldest, GLICKO_PERIOD_MS, 1)).toBe(0); + }); + + it("includes records on the final period boundary", () => { + const newest = oldest + GLICKO_PERIOD_MS; + expect(getGlickoPeriodIndex(newest, oldest, GLICKO_PERIOD_MS, 1)).toBe(0); + }); + + it("splits records across multiple periods", () => { + const numPeriods = 2; + expect(getGlickoPeriodIndex(oldest, oldest, GLICKO_PERIOD_MS, numPeriods)).toBe(0); + expect(getGlickoPeriodIndex(oldest + GLICKO_PERIOD_MS - 1, oldest, GLICKO_PERIOD_MS, numPeriods)).toBe(0); + expect(getGlickoPeriodIndex(oldest + GLICKO_PERIOD_MS, oldest, GLICKO_PERIOD_MS, numPeriods)).toBe(1); + expect(getGlickoPeriodIndex(oldest + 2 * GLICKO_PERIOD_MS, oldest, GLICKO_PERIOD_MS, numPeriods)).toBe(1); + }); +}); + +describe("computeGlickoNumPeriods", () => { + it("returns at least one period", () => { + expect(computeGlickoNumPeriods(0)).toBe(1); + expect(computeGlickoNumPeriods(GLICKO_PERIOD_MS)).toBe(1); + expect(computeGlickoNumPeriods(GLICKO_PERIOD_MS + 1)).toBe(2); + }); +}); + +describe("partitionByGlickoPeriod", () => { + it("assigns every record to exactly one bucket", () => { + const oldestMs = 0; + const records = [ + { dateEndMs: 0 }, + { dateEndMs: GLICKO_PERIOD_MS }, + { dateEndMs: 2 * GLICKO_PERIOD_MS }, + ]; + const numPeriods = computeGlickoNumPeriods(2 * GLICKO_PERIOD_MS); + const buckets = partitionByGlickoPeriod(records, oldestMs, GLICKO_PERIOD_MS, numPeriods); + const assigned = buckets.flat(); + expect(assigned).toHaveLength(records.length); + expect(assigned.map((r) => r.dateEndMs).sort((a, b) => a - b)).toEqual( + records.map((r) => r.dateEndMs).sort((a, b) => a - b), + ); + }); +}); + +describe("medianOf", () => { + it("returns undefined for an empty array", () => { + expect(medianOf([])).toBeUndefined(); + }); + + it("returns the middle value", () => { + expect(medianOf([3, 1, 2])).toBe(2); + expect(medianOf([4, 1, 3, 2])).toBe(2.5); + }); +}); + +describe("percentileOf", () => { + it("interpolates between sorted values", () => { + expect(percentileOf([1, 2, 3, 4, 5], 0)).toBe(1); + expect(percentileOf([1, 2, 3, 4, 5], 100)).toBe(5); + expect(percentileOf([1, 2, 3, 4, 5], 50)).toBe(3); + }); +}); + +describe("computeHoursPerStats", () => { + const earliestMs = 0; + const hourMs = 60 * 60 * 1000; + + it("computes move-weighted mean and per-game median", () => { + const stats = computeHoursPerStats([ + { dateStartMs: 0, dateEndMs: 4 * hourMs, moveSlots: 2 }, + { dateStartMs: 0, dateEndMs: 8 * hourMs, moveSlots: 2 }, + ], earliestMs); + expect(stats.n).toBe(2); + expect(stats.mean).toBe(3); + expect(stats.median).toBe(3); + }); + + it("winsorizes outliers at p2 and p98", () => { + const hourMs = 60 * 60 * 1000; + const games = []; + for (let i = 1; i <= 20; i++) { + games.push({ dateStartMs: 0, dateEndMs: i * hourMs, moveSlots: 1 }); + } + games.push({ dateStartMs: 0, dateEndMs: 10_000 * hourMs, moveSlots: 1 }); + const stats = computeHoursPerStats(games, 0); + expect(stats.n).toBe(21); + expect(stats.winsorizedCount).toBeGreaterThan(0); + const uncappedMean = (Array.from({ length: 20 }, (_, i) => i + 1).reduce((a, b) => a + b, 0) + 10_000) / 21; + expect(stats.mean).toBeLessThan(uncappedMean); + expect(stats.median).toBeLessThan(10_000); + }); + + it("reports zero winsorized records when all rates fall within p2-p98", () => { + const hourMs = 60 * 60 * 1000; + const stats = computeHoursPerStats([ + { dateStartMs: 0, dateEndMs: hourMs, moveSlots: 1 }, + ], 0); + expect(stats.winsorizedCount).toBe(0); + }); + + it("builds weekly medians aligned to completion week buckets", () => { + const weekMs = 7 * 24 * hourMs; + const games = []; + for (let i = 0; i < 5; i++) { + games.push({ dateStartMs: 0, dateEndMs: 2 * hourMs, moveSlots: 1 }); + } + for (let i = 0; i < 5; i++) { + games.push({ dateStartMs: weekMs, dateEndMs: weekMs + 4 * hourMs, moveSlots: 1 }); + } + const stats = computeHoursPerStats(games, earliestMs); + expect(stats.byWeek).toEqual([2, 4]); + }); +}); + +describe("computeReturningPlayersPerWeek", () => { + it("counts users who played again after their first week", () => { + const earliest = 0; + const weekMs = 7 * 24 * 60 * 60 * 1000; + const returning = computeReturningPlayersPerWeek([ + { user: "a", time: 0 }, + { user: "a", time: weekMs }, + { user: "b", time: weekMs }, + ], earliest, 1); + expect(returning).toEqual([0, 1]); + }); +}); + +describe("recordWasPied", () => { + it("detects pied and pie-invoked headers", () => { + expect(recordWasPied({ pied: true } as APGameRecord["header"])).toBe(true); + expect(recordWasPied({ "pie-invoked": true } as APGameRecord["header"])).toBe(true); + expect(recordWasPied({} as APGameRecord["header"])).toBe(false); + }); +}); + +describe("gameSupportsPie", () => { + it("matches pie flags", () => { + expect(gameSupportsPie(["pie"])).toBe(true); + expect(gameSupportsPie(["pie-even"])).toBe(true); + expect(gameSupportsPie(["simultaneous"])).toBe(false); + }); +}); + +describe("gameSupportsMultiPlayerCount", () => { + it("is true when any supported count exceeds two", () => { + expect(gameSupportsMultiPlayerCount([2])).toBe(false); + expect(gameSupportsMultiPlayerCount([2, 3, 4])).toBe(true); + }); +}); + +describe("computeRivalryPairs", () => { + it("counts two-player pairs and filters by minimum games", () => { + const recs = [ + { header: { players: [{ userid: "b" }, { userid: "a" }] } }, + { header: { players: [{ userid: "a" }, { userid: "b" }] } }, + { header: { players: [{ userid: "a" }, { userid: "c" }] } }, + { header: { players: [{ userid: "x" }, { userid: "y" }] } }, + ] as APGameRecord[]; + expect(computeRivalryPairs(recs, 2, 10)).toEqual([ + { userA: "a", userB: "b", n: 2 }, + ]); + }); + + it("filters by minimum games without a top-N cap", () => { + const recs = Array.from({ length: 60 }, () => ({ + header: { players: [{ userid: "a" }, { userid: "b" }] }, + })) as APGameRecord[]; + expect(computeRivalryPairs(recs, 50)).toEqual([ + { userA: "a", userB: "b", n: 60 }, + ]); + expect(computeRivalryPairs(recs, 50, 10)).toEqual([ + { userA: "a", userB: "b", n: 60 }, + ]); + }); + + it("ignores games without two user ids", () => { + const recs = [ + { header: { players: [{ userid: "a" }, { userid: "b" }, { userid: "c" }] } }, + { header: { players: [{ userid: "a" }, {}] } }, + ] as APGameRecord[]; + expect(computeRivalryPairs(recs, 1, 10)).toEqual([]); + }); +}); + +describe("anonymizeRivalries", () => { + it("labels pairs without exposing user ids", () => { + expect(anonymizeRivalries([ + { userA: "secret-a", userB: "secret-b", n: 12 }, + { userA: "secret-c", userB: "secret-d", n: 8 }, + ])).toEqual([ + { rank: 1, label: "Pair 1", n: 12 }, + { rank: 2, label: "Pair 2", n: 8 }, + ]); + }); +}); + +describe("publishRivalries", () => { + const pairs = [ + { userA: "a", userB: "b", n: 12 }, + { userA: "c", userB: "d", n: 8 }, + { userA: "e", userB: "f", n: 5 }, + ]; + const names = new Map([ + ["a", "Alice"], + ["b", "Bob"], + ["c", "Carol"], + ["d", "Dave"], + ["e", "Eve"], + ["f", "Frank"], + ]); + + it("keeps pairs anonymized when neither player opted in", () => { + expect(publishRivalries(pairs, new Set(), names)).toEqual([ + { rank: 1, label: "Pair 1", n: 12 }, + { rank: 2, label: "Pair 2", n: 8 }, + { rank: 3, label: "Pair 3", n: 5 }, + ]); + }); + + it("keeps pairs anonymized when only one player opted in", () => { + expect(publishRivalries(pairs, new Set(["a"]), names)).toEqual([ + { rank: 1, label: "Pair 1", n: 12 }, + { rank: 2, label: "Pair 2", n: 8 }, + { rank: 3, label: "Pair 3", n: 5 }, + ]); + }); + + it("deanonymizes pairs when both players opted in", () => { + expect( + publishRivalries(pairs, new Set(["a", "b", "c", "d"]), names), + ).toEqual([ + { + rank: 1, + label: "Alice vs Bob", + n: 12, + players: [ + { id: "a", name: "Alice" }, + { id: "b", name: "Bob" }, + ], + }, + { + rank: 2, + label: "Carol vs Dave", + n: 8, + players: [ + { id: "c", name: "Carol" }, + { id: "d", name: "Dave" }, + ], + }, + { rank: 3, label: "Pair 3", n: 5 }, + ]); + }); + + it("preserves rank ordering", () => { + const result = publishRivalries(pairs, new Set(["e", "f"]), names); + expect(result.map((r) => r.rank)).toEqual([1, 2, 3]); + expect(result[2]).toEqual({ + rank: 3, + label: "Eve vs Frank", + n: 5, + players: [ + { id: "e", name: "Eve" }, + { id: "f", name: "Frank" }, + ], + }); + }); +}); + +describe("enrichRivalryPairsWithDisplayNames", () => { + it("adds display names and falls back to user id when unknown", () => { + const names = new Map([["a", "Alice"], ["b", "Bob"]]); + expect(enrichRivalryPairsWithDisplayNames([ + { userA: "a", userB: "b", n: 7 }, + { userA: "c", userB: "d", n: 3 }, + ], names)).toEqual([ + { userA: "a", nameA: "Alice", userB: "b", nameB: "Bob", n: 7 }, + { userA: "c", nameA: "c", userB: "d", nameB: "d", n: 3 }, + ]); + }); +}); + +describe("computeTimeoutHistogramRates", () => { + it("returns 0 for empty week buckets instead of NaN", () => { + expect(computeTimeoutHistogramRates([0, 0, 1], [5, 0, 10])).toEqual([0, 0, 0.1]); + }); + + it("aligns arrays of different lengths", () => { + expect(computeTimeoutHistogramRates([1], [2, 4])).toEqual([0.5, 0]); + }); + + it("never produces NaN", () => { + const rates = computeTimeoutHistogramRates([0, 0, 0], [0, 0, 0]); + expect(rates.every((r) => !Number.isNaN(r))).toBe(true); + expect(rates).toEqual([0, 0, 0]); + }); +}); + +describe("recordHasAbandoned", () => { + it("detects plain-string abandoned moves", () => { + const moves: Moves = [["e2-e4", "e7-e5"], ["abandoned"]]; + expect(recordHasAbandoned(moves)).toBe(true); + expect(recordHasTimeout(moves)).toBe(false); + }); + + it("detects structured abandoned moves", () => { + const moves: Moves = [[{ move: "abandoned", result: [{ type: "gameabandoned" }] }]]; + expect(recordHasAbandoned(moves)).toBe(true); + }); + + it("does not false-positive on unrelated move text", () => { + const moves: Moves = [["mention-timeout-in-chat"]]; + expect(recordHasTimeout(moves)).toBe(false); + expect(recordHasAbandoned(moves)).toBe(false); + }); +}); + +describe("recordHasTimeout", () => { + it("detects plain-string timeout moves", () => { + const moves: Moves = [["e2-e4", "timeout"]]; + expect(recordHasTimeout(moves)).toBe(true); + }); + + it("detects structured timeout moves", () => { + const moves: Moves = [[{ move: "timeout", result: [{ type: "timeout", player: 2 }] }]]; + expect(recordHasTimeout(moves)).toBe(true); + }); +}); + +describe("findTimeoutPlayerSeat", () => { + it("uses seat index for full rounds", () => { + const moves: Moves = [["e2-e4", "timeout"]]; + expect(findTimeoutPlayerSeat(moves, 2)).toBe(1); + }); + + it("uses seat index for partial sequential rounds", () => { + const moves: Moves = [["timeout"]]; + expect(findTimeoutPlayerSeat(moves, 2)).toBe(0); + }); + + it("prefers structured result.player over seat index", () => { + const moves: Moves = [[ + { move: "timeout", result: [{ type: "timeout", player: 2 }] }, + null, + ]]; + expect(findTimeoutPlayerSeat(moves, 2)).toBe(1); + }); + + it("returns undefined when no timeout move exists", () => { + const moves: Moves = [["e2-e4", "e7-e5"]]; + expect(findTimeoutPlayerSeat(moves, 2)).toBeUndefined(); + }); +}); + +describe("recordRoundCount / recordMoveSlotCount", () => { + const legacyRec: APGameRecord = { + header: { + game: { name: "Test" }, + site: { name: "Abstract Play", gameid: "legacy-1" }, + "date-start": "2024-01-01T12:00:00Z", + "date-end": "2024-01-01T13:00:00Z", + "date-generated": "2024-01-01T13:00:00Z", + players: [ + { name: "A", result: 1 }, + { name: "B", result: 0 }, + ], + }, + moves: [["e4", "e5"], ["Nf3", "Nc6"], ["Bb5", "a6"]], + }; + + it("legacy records use rec.moves.length and slot sum", () => { + expect(recordRoundCount(legacyRec)).toBe(3); + expect(recordMoveSlotCount(legacyRec)).toBe(6); + }); + + it("skip-turn header counts non-empty rounds and slots only", () => { + const rec: APGameRecord = { + ...legacyRec, + header: { + ...legacyRec.header, + "turn-model": "skip-turn", + }, + moves: [ + ["m1", null], + [null, null], + ["m2", null], + ], + }; + expect(recordRoundCount(rec)).toBe(2); + expect(recordMoveSlotCount(rec)).toBe(2); + }); +}); + +describe("record gameid helpers", () => { + const INSTANCE_ID = "f47ac10b-58cc-4372-a567-0e02b2c3d479"; + + function recWithGameid(gameid: string): APGameRecord { + return { + header: { + game: { name: "Go", variants: ["9x9 board"] }, + site: { name: "Abstract Play", gameid }, + "date-start": "2024-01-01T12:00:00Z", + "date-end": "2024-01-01T13:00:00Z", + "date-generated": "2024-01-01T13:00:00Z", + players: [ + { name: "A", result: 1 }, + { name: "B", result: 0 }, + ], + }, + moves: [["e4", "e5"]], + }; + } + + it("reads meta UID and variant codes from encoded gameids", () => { + const rec = recWithGameid(`${INSTANCE_ID}#go:size-9`); + expect(metaGameFromRecord(rec)).toBe("go"); + expect(variantUidsFromRecord(rec)).to.include("size-9"); + expect(variantComboFromRecord(rec)).toContain("size-9"); + }); + + it("canonicalizes akimbo empty gameid variants like size-13", () => { + const emptyRec = recWithGameid(`${INSTANCE_ID}#akimbo:`); + emptyRec.header.game.name = "Akimbo"; + const sizedRec = recWithGameid(`${INSTANCE_ID}#akimbo:size-13`); + sizedRec.header.game.name = "Akimbo"; + expect(variantComboFromRecord(emptyRec)).to.equal(variantComboFromRecord(sizedRec)); + }); + + it("reads meta UID from legacy gameids", () => { + const rec = recWithGameid(`go#${INSTANCE_ID}`); + const legacy = { onLegacyGameId: vi.fn(), onLegacyVariantFallback: vi.fn() }; + expect(metaGameFromRecord(rec, legacy)).toBe("go"); + expect(legacy.onLegacyGameId).toHaveBeenCalled(); + expect(variantUidsFromRecord(rec, legacy).length).toBeGreaterThan(0); + expect(legacy.onLegacyVariantFallback).toHaveBeenCalled(); + }); +}); diff --git a/crons/src/functions/summarizeHelpers.ts b/crons/src/functions/summarizeHelpers.ts new file mode 100644 index 00000000..2528d09a --- /dev/null +++ b/crons/src/functions/summarizeHelpers.ts @@ -0,0 +1,1104 @@ +import type { APGameRecord } from "@abstractplay/recranks"; +import type { + GlickoByGameRow, + GlickoGameCounts, + GlickoSiteCounts, + GlickoSiteEntry, + GlickoStats, +} from "types/stats/GlickoStats.js"; +import type { PlayerTimeoutStats } from "types/stats/PlayerTimeoutStats.js"; +import type { StatSummary } from "types/stats/StatSummary.js"; +import type { + PlayerSummarySlice, + StatSummaryPlayers, + StatSummaryRatings, + StatSummarySite, +} from "types/stats/StatSummaryTiers.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; +import type { UserNumList } from "types/stats/UserNumList.js"; +import type { UserNumber } from "types/stats/UserNumber.js"; +import type { TwoPlayerStats } from "types/stats/TwoPlayerStats.js"; +import { gameinfo, variantUidsForBatchRating } from "@abstractplay/gameslib"; +import { parseRecordGameId, variantComboKey } from "../utils/recordGameId.js"; + +export const GLICKO_PERIOD_MS = 60 * 24 * 60 * 60 * 1000; +export const GLICKO_RATING_START = 1200; +export const GLICKO_RD_START = 350; +export const GLICKO_VOLATILITY_START = 0.06; + +type MoveSlot = APGameRecord["moves"][number][number]; +type TurnModel = "sequential" | "simultaneous" | "sequenced" | "skip-turn"; +const TURN_MODELS: TurnModel[] = ["sequential", "simultaneous", "sequenced", "skip-turn"]; + +const turnModelFromRecord = (rec: APGameRecord): TurnModel | undefined => { + const raw = rec.header["turn-model"]; + if (typeof raw === "string" && (TURN_MODELS as string[]).includes(raw)) { + return raw as TurnModel; + } + return undefined; +}; + +const slotMoveText = (slot: MoveSlot): string | undefined => { + if (slot === null) { + return undefined; + } + if (typeof slot === "string") { + return slot; + } + return slot.move; +}; + +const isEmptyMoveSlot = (slot: MoveSlot): boolean => { + if (slot === null) { + return true; + } + const text = slotMoveText(slot); + return text === undefined || text === ""; +}; + +const roundHasRealMove = (round: APGameRecord["moves"][number]): boolean => + round.some((slot) => !isEmptyMoveSlot(slot)); + +/** Returns Math.max of nums, or -1 when empty (so `i <= maxBucket` loops are no-ops). */ +export function maxOf(nums: number[]): number { + return nums.length > 0 ? Math.max(...nums) : -1; +} + +export function isTimeoutSlot(m: MoveSlot): boolean { + return m !== null && (typeof m === "object" ? m.move === "timeout" : m === "timeout"); +} + +export function isAbandonedSlot(m: MoveSlot): boolean { + return m !== null && (typeof m === "object" ? m.move === "abandoned" : m === "abandoned"); +} + +export function recordHasAbandoned(moves: APGameRecord["moves"]): boolean { + return moves.some((round) => round.some(isAbandonedSlot)); +} + +export function recordHasTimeout(moves: APGameRecord["moves"]): boolean { + return moves.some((round) => round.some(isTimeoutSlot)); +} + +export function findTimeoutPlayerSeat(moves: APGameRecord["moves"], numPlayers: number): number | undefined { + const roundIdx = moves.findIndex((round) => round.some(isTimeoutSlot)); + if (roundIdx === -1) { + return undefined; + } + const round = moves[roundIdx]; + const seatIdx = round.findIndex(isTimeoutSlot); + if (seatIdx === -1) { + return undefined; + } + const slot = round[seatIdx]; + if (slot === null) { + return undefined; + } + if (typeof slot === "object" && slot.result !== undefined) { + const results = Array.isArray(slot.result) ? slot.result : [slot.result]; + for (const r of results) { + if ( + typeof r === "object" && + r !== null && + "type" in r && + r.type === "timeout" && + "player" in r && + typeof r.player === "number" + ) { + return r.player - 1; + } + } + } + if (round.length === numPlayers) { + return seatIdx; + } + return seatIdx; +} + +export function getGlickoPeriodIndex( + secs: number, + oldestMs: number, + periodMs: number, + numPeriods: number, +): number { + if (numPeriods <= 1) { + return 0; + } + const idx = Math.floor((secs - oldestMs) / periodMs); + return Math.min(numPeriods - 1, Math.max(0, idx)); +} + +export function computeGlickoNumPeriods(deltaMs: number, periodMs: number = GLICKO_PERIOD_MS): number { + let numPeriods = Math.ceil(deltaMs / periodMs); + if (numPeriods === 0) { + numPeriods++; + } + return numPeriods; +} + +export function partitionByGlickoPeriod( + records: T[], + oldestMs: number, + periodMs: number, + numPeriods: number, +): T[][] { + const buckets: T[][] = Array.from({ length: numPeriods }, () => []); + for (const rec of records) { + const period = getGlickoPeriodIndex(rec.dateEndMs, oldestMs, periodMs, numPeriods); + buckets[period].push(rec); + } + return buckets; +} + +const MS_PER_HOUR = 60 * 60 * 1000; +const MS_PER_DAY = 24 * MS_PER_HOUR; +const HOURS_PER_WINSORIZE_LOW = 2; +const HOURS_PER_WINSORIZE_HIGH = 98; + +export function percentileOf(nums: number[], p: number): number | undefined { + if (nums.length === 0) { + return undefined; + } + const sorted = [...nums].sort((a, b) => a - b); + const idx = (p / 100) * (sorted.length - 1); + const lower = Math.floor(idx); + const upper = Math.ceil(idx); + if (lower === upper) { + return sorted[lower]; + } + return sorted[lower] + (sorted[upper] - sorted[lower]) * (idx - lower); +} + +export function medianOf(nums: number[]): number | undefined { + if (nums.length === 0) { + return undefined; + } + const sorted = [...nums].sort((a, b) => a - b); + if (sorted.length % 2 === 0) { + const rightIdx = sorted.length / 2; + const leftIdx = rightIdx - 1; + return (sorted[leftIdx] + sorted[rightIdx]) / 2; + } + return sorted[Math.floor(sorted.length / 2)]; +} + +export type HoursPerGameInput = { + dateStartMs: number; + dateEndMs: number; + moveSlots: number; +}; + +export type HoursPerStatsResult = { + mean: number; + median: number; + n: number; + byWeek: number[]; + winsorizedCount: number; +}; + +type HoursPerGameComputed = HoursPerGameInput & { + hours: number; + bucket: number; +}; + +export function computeHoursPerStats( + games: HoursPerGameInput[], + earliestMs: number, +): HoursPerStatsResult { + const computed: HoursPerGameComputed[] = []; + for (const game of games) { + if (game.moveSlots <= 0) { + continue; + } + const duration = game.dateEndMs - game.dateStartMs; + const hours = (duration / game.moveSlots) / MS_PER_HOUR; + const daysAgo = (game.dateEndMs - earliestMs) / MS_PER_DAY; + const bucket = Math.floor(daysAgo / 7); + computed.push({ ...game, hours, bucket }); + } + + const rawRates = computed.map((g) => g.hours); + const pLow = percentileOf(rawRates, HOURS_PER_WINSORIZE_LOW); + const pHigh = percentileOf(rawRates, HOURS_PER_WINSORIZE_HIGH); + let winsorizedCount = 0; + let totalMoveSlots = 0; + let weightedHoursSum = 0; + const winsorizedRates: number[] = []; + const byWeekBuckets = new Map(); + + for (const game of computed) { + let rate = game.hours; + if (pLow !== undefined && rate < pLow) { + winsorizedCount++; + rate = pLow; + } else if (pHigh !== undefined && rate > pHigh) { + winsorizedCount++; + rate = pHigh; + } + winsorizedRates.push(rate); + totalMoveSlots += game.moveSlots; + weightedHoursSum += rate * game.moveSlots; + const lst = byWeekBuckets.get(game.bucket); + if (lst === undefined) { + byWeekBuckets.set(game.bucket, [rate]); + } else { + lst.push(rate); + } + } + + const mean = totalMoveSlots > 0 ? weightedHoursSum / totalMoveSlots : 0; + const median = medianOf(winsorizedRates) ?? 0; + const maxBucket = maxOf([...byWeekBuckets.keys()]); + const byWeek: number[] = []; + for (let i = 0; i <= maxBucket; i++) { + byWeek.push(medianOf(byWeekBuckets.get(i) ?? []) ?? 0); + } + + return { + mean, + median, + n: winsorizedRates.length, + byWeek, + winsorizedCount, + }; +} + +export type WeekActivity = { + user: string; + time: number; +}; + +export function recordWasPied(header: APGameRecord["header"]): boolean { + if (header.pied === true) { + return true; + } + const pieInvoked = (header as { "pie-invoked"?: boolean })["pie-invoked"]; + return pieInvoked === true; +} + +export function gameSupportsPie(flags: string[] | undefined): boolean { + if (flags === undefined) { + return false; + } + return flags.includes("pie") || flags.includes("pie-even"); +} + +export function gameSupportsMultiPlayerCount(playercounts: number[]): boolean { + return playercounts.some((n) => n > 2); +} + +export function computeReturningPlayersPerWeek( + activities: WeekActivity[], + earliestMs: number, + maxBucket: number, +): number[] { + const userFirstBucket = new Map(); + const userPlayBuckets = new Map>(); + + for (const { user, time } of activities) { + const bucket = Math.floor((time - earliestMs) / MS_PER_DAY / 7); + const prev = userFirstBucket.get(user); + if (prev === undefined || bucket < prev) { + userFirstBucket.set(user, bucket); + } + let set = userPlayBuckets.get(user); + if (set === undefined) { + set = new Set(); + userPlayBuckets.set(user, set); + } + set.add(bucket); + } + + const returningPlayers: number[] = []; + for (let i = 0; i <= maxBucket; i++) { + let count = 0; + for (const [user, buckets] of userPlayBuckets.entries()) { + if (buckets.has(i) && (userFirstBucket.get(user) ?? i) < i) { + count++; + } + } + returningPlayers.push(count); + } + return returningPlayers; +} + +export const RIVALRY_MIN_GAMES = 5; +export const RIVALRY_PUBLIC_MIN_GAMES = 50; + +export function pairKey(userA: string, userB: string): string { + return userA < userB ? `${userA}|${userB}` : `${userB}|${userA}`; +} + +export type RivalryPairResult = { + userA: string; + userB: string; + n: number; +}; + +export function computeRivalryPairs( + recs: APGameRecord[], + minGames: number = RIVALRY_MIN_GAMES, + topN?: number, +): RivalryPairResult[] { + const counts = new Map(); + for (const rec of recs) { + accumulateRivalryPair(counts, rec); + } + return finalizeRivalryPairs(counts, minGames, topN); +} + +export function accumulateRivalryPair( + counts: Map, + rec: APGameRecord, +): void { + if (rec.header.players.length !== 2) { + return; + } + const p0 = rec.header.players[0].userid; + const p1 = rec.header.players[1].userid; + if (p0 === undefined || p1 === undefined) { + return; + } + const userA = p0 < p1 ? p0 : p1; + const userB = p0 < p1 ? p1 : p0; + const key = pairKey(userA, userB); + const existing = counts.get(key); + if (existing === undefined) { + counts.set(key, { userA, userB, n: 1 }); + } else { + existing.n++; + } +} + +export function finalizeRivalryPairs( + counts: Map, + minGames: number = RIVALRY_MIN_GAMES, + topN?: number, +): RivalryPairResult[] { + const sorted = [...counts.values()] + .filter((p) => p.n >= minGames) + .sort((a, b) => b.n - a.n || a.userA.localeCompare(b.userA) || a.userB.localeCompare(b.userB)); + if (topN === undefined) { + return sorted; + } + return sorted.slice(0, topN); +} + +export type AnonymizedRivalryResult = { + rank: number; + label: string; + n: number; + players?: { + id: string; + name: string; + }[]; +}; + +export function anonymizeRivalries(pairs: RivalryPairResult[]): AnonymizedRivalryResult[] { + return pairs.map((p, i) => ({ + rank: i + 1, + label: `Pair ${i + 1}`, + n: p.n, + })); +} + +export function publishRivalries( + pairs: RivalryPairResult[], + publicUserIds: Set, + displayNames: Map, +): AnonymizedRivalryResult[] { + return pairs.map((p, i) => { + const rank = i + 1; + if (publicUserIds.has(p.userA) && publicUserIds.has(p.userB)) { + const nameA = displayNames.get(p.userA) ?? p.userA; + const nameB = displayNames.get(p.userB) ?? p.userB; + return { + rank, + label: `${nameA} vs ${nameB}`, + n: p.n, + players: [ + { id: p.userA, name: nameA }, + { id: p.userB, name: nameB }, + ], + }; + } + return { + rank, + label: `Pair ${rank}`, + n: p.n, + }; + }); +} + +export type IdentifiedRivalryPairResult = { + userA: string; + nameA: string; + userB: string; + nameB: string; + n: number; +}; + +export function enrichRivalryPairsWithDisplayNames( + pairs: RivalryPairResult[], + displayNames: Map, +): IdentifiedRivalryPairResult[] { + return pairs.map((p) => ({ + userA: p.userA, + nameA: displayNames.get(p.userA) ?? p.userA, + userB: p.userB, + nameB: displayNames.get(p.userB) ?? p.userB, + n: p.n, + })); +} + +export function computeTimeoutHistogramRates(histTimeouts: number[], histAll: number[]): number[] { + const len = Math.max(histTimeouts.length, histAll.length); + const rates: number[] = []; + for (let i = 0; i < len; i++) { + const timeouts = histTimeouts[i] ?? 0; + const all = histAll[i] ?? 0; + rates.push(all > 0 ? timeouts / all : 0); + } + return rates; +} + +/** Gamerecord round count — legacy `rec.moves.length` when no `turn-model` header. */ +export function recordRoundCount(rec: APGameRecord): number { + const model = turnModelFromRecord(rec); + if (model === undefined) { + return rec.moves?.length ?? 0; + } + return rec.moves.filter((round) => roundHasRealMove(round)).length; +} + +export type RecordGameIdFallback = { + resolveMetaUidFromDisplayName?: (displayName: string) => string | undefined; + onLegacyGameId?: () => void; + onLegacyVariantFallback?: () => void; +}; + +export function metaGameFromRecord( + rec: APGameRecord, + fallback?: RecordGameIdFallback, +): string { + const gameid = rec.header.site?.gameid; + if (gameid !== undefined) { + const parsed = parseRecordGameId(gameid); + if (parsed !== undefined) { + if (parsed.legacy) { + fallback?.onLegacyGameId?.(); + } + return parsed.metaGame; + } + } + const fromName = fallback?.resolveMetaUidFromDisplayName?.(rec.header.game.name); + if (fromName !== undefined) { + fallback?.onLegacyGameId?.(); + return fromName; + } + return rec.header.game.name; +} + +function rawVariantUidsFromRecord( + rec: APGameRecord, + fallback?: RecordGameIdFallback, +): string[] { + const gameid = rec.header.site?.gameid; + if (gameid !== undefined) { + const parsed = parseRecordGameId(gameid); + if (parsed !== undefined && !parsed.legacy) { + return parsed.variantUids; + } + } + fallback?.onLegacyVariantFallback?.(); + if (rec.header.game.variants !== undefined && rec.header.game.variants.length > 0) { + return [...rec.header.game.variants].sort(); + } + return []; +} + +export function variantUidsFromRecord( + rec: APGameRecord, + fallback?: RecordGameIdFallback, +): string[] { + const raw = rawVariantUidsFromRecord(rec, fallback); + const metaUid = metaGameFromRecord(rec, fallback); + const defs = gameinfo.get(metaUid)?.variants; + if (defs === undefined || defs.length === 0) { + return raw; + } + const playerCount = rec.header.players.length > 0 ? rec.header.players.length : 2; + return variantUidsForBatchRating(metaUid, playerCount, raw); +} + +export function variantComboFromRecord( + rec: APGameRecord, + fallback?: RecordGameIdFallback, +): string { + return variantComboKey(variantUidsFromRecord(rec, fallback)); +} + +/** @deprecated Prefer variantComboFromRecord for stable variant UID grouping. */ +export function sortVariants(rec: APGameRecord): string { + if (rec.header.game.variants !== undefined && rec.header.game.variants.length > 0) { + const lst = [...rec.header.game.variants]; + lst.sort(); + return lst.join("|"); + } + return ""; +} + +export function calcTwoPlayerStats(recs: APGameRecord[]): TwoPlayerStats | undefined { + let n = 0; + let fpWins = 0; + let draws = 0; + const lengths: number[] = []; + for (const rec of recs) { + if (rec.header.players.length === 2 && recordRoundCount(rec) > 2) { + n++; + lengths.push(recordRoundCount(rec)); + if (rec.header.players[0].result > rec.header.players[1].result) { + fpWins++; + } else if (rec.header.players[0].result === rec.header.players[1].result) { + fpWins += 0.5; + draws++; + } + } + } + if (n === 0) { + return undefined; + } + const wins = fpWins / n; + const sum = lengths.reduce((prev, curr) => prev + curr, 0); + const avg = sum / lengths.length; + lengths.sort((a, b) => a - b); + let median: number; + if (lengths.length % 2 === 0) { + const rightIdx = lengths.length / 2; + const leftIdx = rightIdx - 1; + median = (lengths[leftIdx] + lengths[rightIdx]) / 2; + } else { + median = lengths[Math.floor(lengths.length / 2)]; + } + return { + n, + lenAvg: avg, + lenMedian: median, + winsFirst: wins, + drawRate: draws / n, + }; +} + +export function hIndexFromCounts(counts: Iterable): number { + const sorted = [...counts].sort((a, b) => b - a); + let index = sorted.length; + for (let i = 0; i < sorted.length; i++) { + if (sorted[i]! < i + 1) { + index = i; + break; + } + } + return index; +} + +/** Total move slots for hours-per-move — legacy sum of round widths when no header. */ +export function recordMoveSlotCount(rec: APGameRecord): number { + const model = turnModelFromRecord(rec); + if (model === undefined) { + return rec.moves.reduce((sum, round) => sum + round.length, 0); + } + let total = 0; + for (const round of rec.moves) { + for (const slot of round) { + if (!isEmptyMoveSlot(slot)) { + total++; + } + } + } + return total; +} + +export const GLICKO_ESTABLISHED_RD = 110; +export const GLICKO_PROVISIONAL_RD = 200; +export const GLICKO_MIN_GAMES_ESTABLISHED = 20; +export const GLICKO_MIN_GAMES_PROVISIONAL = 10; + +export const isGlickoProvisional = (rd: number, n: number): boolean => + n < GLICKO_MIN_GAMES_PROVISIONAL || rd > GLICKO_PROVISIONAL_RD; + +export const isGlickoEstablished = (rd: number, n: number): boolean => + n >= GLICKO_MIN_GAMES_ESTABLISHED && rd <= GLICKO_ESTABLISHED_RD; + +export function toGlickoStats(rating: number, rd: number, volatility: number, n: number): GlickoStats { + const ratingLow = rating - 2 * rd; + const ratingHigh = rating + 2 * rd; + return { + rating, + rd, + volatility, + ratingLow, + ratingHigh, + provisional: isGlickoProvisional(rd, n), + established: isGlickoEstablished(rd, n), + n, + }; +} + +export function buildGlickoByGame(rows: { user: string; game: string; glicko: GlickoStats }[]): GlickoByGameRow[] { + return rows.map((row) => ({ user: row.user, game: row.game, glicko: row.glicko })); +} + +export function computeGlickoSiteRatings(rows: GlickoByGameRow[]): GlickoSiteEntry[] { + const byUser = new Map(); + for (const row of rows) { + const list = byUser.get(row.user); + if (list === undefined) { + byUser.set(row.user, [row]); + } else { + list.push(row); + } + } + const site: GlickoSiteEntry[] = []; + for (const [user, userRows] of byUser.entries()) { + let totalN = 0; + let weightedRating = 0; + let weightedRd = 0; + let weightedRatingLow = 0; + let weightedRatingHigh = 0; + let provisional = false; + let established = false; + for (const row of userRows) { + const { glicko } = row; + totalN += glicko.n; + weightedRating += glicko.rating * glicko.n; + weightedRd += glicko.rd * glicko.n; + weightedRatingLow += glicko.ratingLow * glicko.n; + weightedRatingHigh += glicko.ratingHigh * glicko.n; + provisional = provisional || glicko.provisional; + established = established || glicko.established; + } + if (totalN <= 0) { + continue; + } + site.push({ + user, + rating: weightedRating / totalN, + rd: weightedRd / totalN, + ratingLow: weightedRatingLow / totalN, + ratingHigh: weightedRatingHigh / totalN, + n: totalN, + provisional, + established, + }); + } + site.sort((a, b) => a.user.localeCompare(b.user)); + return site; +} + +export function computeGlickoGameCounts(rows: GlickoByGameRow[]): GlickoGameCounts[] { + const byGame = new Map(); + for (const row of rows) { + let counts = byGame.get(row.game); + if (counts === undefined) { + counts = { game: row.game, rated: 0, provisional: 0, established: 0 }; + byGame.set(row.game, counts); + } + counts.rated++; + if (row.glicko.provisional) { + counts.provisional++; + } + if (row.glicko.established) { + counts.established++; + } + } + return [...byGame.values()].sort((a, b) => a.game.localeCompare(b.game)); +} + +export function computeGlickoSiteCounts(site: GlickoSiteEntry[]): GlickoSiteCounts { + let provisional = 0; + let established = 0; + for (const entry of site) { + if (entry.provisional) { + provisional++; + } + if (entry.established) { + established++; + } + } + return { + rated: site.length, + provisional, + established, + }; +} + +export type PlayerTimeoutAccumulator = { + count: number; + latestTimeoutMs: number; + times: number[]; +}; + +export function recordPlayerTimeout( + acc: Map, + user: string, + dateMs: number, +): void { + let entry = acc.get(user); + if (entry === undefined) { + entry = { count: 0, latestTimeoutMs: 0, times: [] }; + acc.set(user, entry); + } + entry.count++; + entry.latestTimeoutMs = Math.max(entry.latestTimeoutMs, dateMs); + entry.times.push(dateMs); +} + +export function timeoutStatsFromAccumulator( + acc: Map, +): PlayerTimeoutStats[] { + return [...acc.entries()] + .map(([user, entry]) => ({ + user, + count: entry.count, + latestTimeoutMs: entry.latestTimeoutMs, + })) + .sort((a, b) => a.user.localeCompare(b.user)); +} + +export function buildPlayerTimeoutHistograms( + timeoutAcc: Map, + allUserIds: Iterable, + earliestMs: number, +): UserNumList[] { + const histPlayerTimeouts: UserNumList[] = []; + for (const userid of allUserIds) { + const entry = timeoutAcc.get(userid); + const buckets: { bucket: number }[] = []; + if (entry !== undefined) { + for (const value of entry.times) { + const daysAgo = (value - earliestMs) / (24 * 60 * 60 * 1000); + const bucket = Math.floor(daysAgo / 7); + buckets.push({ bucket }); + } + } + const maxBucket = maxOf(buckets.map((x) => x.bucket)); + const lst: number[] = []; + for (let i = 0; i <= maxBucket; i++) { + lst.push(buckets.filter((x) => x.bucket === i).length); + } + histPlayerTimeouts.push({ user: userid, value: [...lst] }); + } + return histPlayerTimeouts; +} + +export function splitStatSummary( + summary: StatSummary, + generated: string, +): { site: StatSummarySite; players: StatSummaryPlayers; ratings: StatSummaryRatings } { + return { + site: { + generated, + tier: "site", + numGames: summary.numGames, + numPlayers: summary.numPlayers, + oldestRec: summary.oldestRec, + newestRec: summary.newestRec, + timeoutRate: summary.timeoutRate, + abandonedRate: summary.abandonedRate, + playContext: summary.playContext, + pieRates: summary.pieRates, + playerCountMix: summary.playerCountMix, + geoStats: summary.geoStats, + activeGeoStats: summary.activeGeoStats, + seasonality: summary.seasonality, + rivalries: summary.rivalries, + hoursPer: summary.hoursPer, + recent: summary.recent, + histograms: { + all: summary.histograms.all, + allPlayers: summary.histograms.allPlayers, + activeMovers: summary.histograms.activeMovers, + returningPlayers: summary.histograms.returningPlayers, + firstTimers: summary.histograms.firstTimers, + timeouts: summary.histograms.timeouts, + abandoned: summary.histograms.abandoned, + meta: summary.histograms.meta, + }, + hMeta: summary.hMeta, + metaStats: summary.metaStats, + soloMetaStats: summary.soloMetaStats, + soloSeedBoards: summary.soloSeedBoards, + plays: summary.plays, + topPlayers: summary.topPlayers, + }, + players: { + generated, + tier: "players", + players: summary.players, + histograms: { + players: summary.histograms.players, + playerTimeouts: summary.histograms.playerTimeouts, + }, + pastDisplayNames: summary.pastDisplayNames, + }, + ratings: { + generated, + tier: "ratings", + ratings: summary.ratings, + }, + }; +} + +const userNumberMap = (list: UserNumber[]): Map => + new Map(list.map((row) => [row.user, row.value])); + +export type PlayerSummaryIndexes = { + allPlays: Map; + eclectic: Map; + social: Map; + h: Map; + hOpp: Map; + timeoutStats: Map; + histPlayers: Map; + histPlayerTimeouts: Map; + highest: Map; + glickoByGame: Map; + glickoSite: Map; + avg: Map; + weighted: Map; + pastDisplayNames: Map; +}; + +export function buildPlayerSummaryIndexesFromTiers( + playersTier: StatSummaryPlayers, + ratingsTier: StatSummaryRatings, +): PlayerSummaryIndexes { + const highest = new Map(); + for (const row of ratingsTier.ratings.highest) { + const list = highest.get(row.user); + if (list === undefined) { + highest.set(row.user, [row]); + } else { + list.push(row); + } + } + const glickoByGame = new Map(); + for (const row of ratingsTier.ratings.glickoByGame) { + const list = glickoByGame.get(row.user); + if (list === undefined) { + glickoByGame.set(row.user, [row]); + } else { + list.push(row); + } + } + return { + allPlays: userNumberMap(playersTier.players.allPlays), + eclectic: userNumberMap(playersTier.players.eclectic), + social: userNumberMap(playersTier.players.social), + h: userNumberMap(playersTier.players.h), + hOpp: userNumberMap(playersTier.players.hOpp), + timeoutStats: new Map(playersTier.players.timeoutStats.map((row) => [row.user, row])), + histPlayers: new Map(playersTier.histograms.players.map((row) => [row.user, row.value])), + histPlayerTimeouts: new Map(playersTier.histograms.playerTimeouts.map((row) => [row.user, row.value])), + highest, + glickoByGame, + glickoSite: new Map(ratingsTier.ratings.glickoSite.map((row) => [row.user, row])), + avg: new Map(ratingsTier.ratings.avg.map((row) => [row.user, row.rating])), + weighted: new Map(ratingsTier.ratings.weighted.map((row) => [row.user, row.rating])), + pastDisplayNames: new Map( + (playersTier.pastDisplayNames ?? []).map((row) => [row.user, row.names]), + ), + }; +} + +export function buildPlayerSummaryIndexes(summary: StatSummary): PlayerSummaryIndexes { + return buildPlayerSummaryIndexesFromTiers( + { + generated: "", + tier: "players", + players: summary.players, + histograms: { + players: summary.histograms.players, + playerTimeouts: summary.histograms.playerTimeouts, + }, + pastDisplayNames: summary.pastDisplayNames, + }, + { + generated: "", + tier: "ratings", + ratings: summary.ratings, + }, + ); +} + +export function collectPlayerSummaryUserIdsFromTiers( + playersTier: StatSummaryPlayers, + ratingsTier: StatSummaryRatings, +): string[] { + const users = new Set(); + for (const row of playersTier.players.allPlays) { + users.add(row.user); + } + for (const row of playersTier.players.eclectic) { + users.add(row.user); + } + for (const row of playersTier.players.social) { + users.add(row.user); + } + for (const row of playersTier.players.h) { + users.add(row.user); + } + for (const row of playersTier.players.hOpp) { + users.add(row.user); + } + for (const row of playersTier.players.timeoutStats) { + users.add(row.user); + } + for (const row of ratingsTier.ratings.highest) { + users.add(row.user); + } + return [...users].sort((a, b) => a.localeCompare(b)); +} + +export function collectPlayerSummaryUserIds(summary: StatSummary): string[] { + return collectPlayerSummaryUserIdsFromTiers( + { + generated: "", + tier: "players", + players: summary.players, + histograms: { + players: summary.histograms.players, + playerTimeouts: summary.histograms.playerTimeouts, + }, + }, + { + generated: "", + tier: "ratings", + ratings: summary.ratings, + }, + ); +} + +export function toPlayerSummarySlice( + user: string, + generated: string, + indexes: PlayerSummaryIndexes, +): PlayerSummarySlice { + const players: PlayerSummarySlice["players"] = {}; + const allPlays = indexes.allPlays.get(user); + if (allPlays !== undefined) { + players.allPlays = allPlays; + } + const eclectic = indexes.eclectic.get(user); + if (eclectic !== undefined) { + players.eclectic = eclectic; + } + const social = indexes.social.get(user); + if (social !== undefined) { + players.social = social; + } + const h = indexes.h.get(user); + if (h !== undefined) { + players.h = h; + } + const hOpp = indexes.hOpp.get(user); + if (hOpp !== undefined) { + players.hOpp = hOpp; + } + const timeout = indexes.timeoutStats.get(user); + if (timeout !== undefined) { + players.timeoutCount = timeout.count; + players.latestTimeoutMs = timeout.latestTimeoutMs; + } + + const histograms: PlayerSummarySlice["histograms"] = {}; + const playerHist = indexes.histPlayers.get(user); + if (playerHist !== undefined) { + histograms.players = playerHist; + } + const timeoutHist = indexes.histPlayerTimeouts.get(user); + if (timeoutHist !== undefined) { + histograms.playerTimeouts = timeoutHist; + } + + const ratings: PlayerSummarySlice["ratings"] = { + highest: indexes.highest.get(user) ?? [], + }; + const glickoRows = indexes.glickoByGame.get(user); + if (glickoRows !== undefined && glickoRows.length > 0) { + ratings.glickoByGame = glickoRows; + } + const glickoSite = indexes.glickoSite.get(user); + if (glickoSite !== undefined) { + ratings.glickoSite = glickoSite; + } + const avg = indexes.avg.get(user); + if (avg !== undefined) { + ratings.avg = avg; + } + const weighted = indexes.weighted.get(user); + if (weighted !== undefined) { + ratings.weighted = weighted; + } + + const pastDisplayNames = indexes.pastDisplayNames.get(user); + return { + generated, + user, + ...(pastDisplayNames !== undefined && pastDisplayNames.length > 0 + ? { pastDisplayNames } + : {}), + players, + histograms, + ratings, + }; +} + +/** Keys present on the monolith that are partitioned across tier files. */ +export const STAT_SUMMARY_PARTITIONED_KEYS = [ + "numGames", + "numPlayers", + "oldestRec", + "newestRec", + "timeoutRate", + "abandonedRate", + "playContext", + "pieRates", + "playerCountMix", + "geoStats", + "activeGeoStats", + "seasonality", + "rivalries", + "hoursPer", + "recent", + "hMeta", + "metaStats", + "soloMetaStats", + "soloSeedBoards", + "plays", + "topPlayers", + "players", + "pastDisplayNames", + "ratings", + "histograms", +] as const; + +export function statSummaryTierKeys(site: StatSummarySite, players: StatSummaryPlayers, ratings: StatSummaryRatings): Set { + const keys = new Set(); + for (const key of Object.keys(site)) { + if (key !== "generated" && key !== "tier") { + keys.add(key); + } + } + for (const key of Object.keys(players)) { + if (key !== "generated" && key !== "tier") { + keys.add(key); + } + } + for (const key of Object.keys(ratings)) { + if (key !== "generated" && key !== "tier") { + keys.add(key); + } + } + return keys; +} diff --git a/crons/src/functions/summarizeMeta.ts b/crons/src/functions/summarizeMeta.ts new file mode 100644 index 00000000..525b718c --- /dev/null +++ b/crons/src/functions/summarizeMeta.ts @@ -0,0 +1,262 @@ +import { GetObjectCommand, ListObjectsV2Command, type S3Client } from "@aws-sdk/client-s3"; +import { + type APGameRecord, + ELOBasic, + Glicko2, + type IGlickoRating, + type IRating, + type ITrueskillRating, + Trueskill, +} from "@abstractplay/recranks"; +import { replacer } from "@abstractplay/gameslib"; +import type { TwoPlayerStats } from "types/stats/TwoPlayerStats.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; +import type { UserNumber } from "types/stats/UserNumber.js"; +import { + GLICKO_PERIOD_MS, + GLICKO_RATING_START, + GLICKO_RD_START, + calcTwoPlayerStats, + computeGlickoNumPeriods, + hIndexFromCounts, + partitionByGlickoPeriod, + variantComboFromRecord, + type RecordGameIdFallback, + toGlickoStats, +} from "./summarizeHelpers.js"; +import { batchRatingGameLabel } from "../lib/batchRatings.js"; + +export type RatingListEntry = { + user: string; + game: string; + rating: IRating; +}; + +export async function listMetaShardKeys(s3: S3Client, bucket: string): Promise { + const keys: string[] = []; + let continuationToken: string | undefined; + do { + const response = await s3.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: "meta/", + ContinuationToken: continuationToken, + })); + for (const obj of response.Contents ?? []) { + const key = obj.Key; + if (key === undefined || !key.endsWith(".json")) { + continue; + } + const metaUid = key.slice("meta/".length, -".json".length); + if (metaUid.length > 0) { + keys.push(metaUid); + } + } + continuationToken = response.IsTruncated ? response.NextContinuationToken : undefined; + } while (continuationToken !== undefined); + keys.sort(); + return keys; +} + +export async function loadMetaShard( + s3: S3Client, + bucket: string, + metaUid: string, +): Promise { + const response = await s3.send(new GetObjectCommand({ + Bucket: bucket, + Key: `meta/${metaUid}.json`, + })); + const str = await response.Body?.transformToString(); + if (str === undefined) { + throw new Error(`Unable to load meta/${metaUid}.json`); + } + return JSON.parse(str) as APGameRecord[]; +} + +export function buildMetaStatsForGame( + recs: APGameRecord[], + metaUid: string, + fallback?: RecordGameIdFallback, +): Record { + const metaStats: Record = {}; + const combined = calcTwoPlayerStats(recs); + if (combined !== undefined) { + metaStats[metaUid] = combined; + } + const allVariants = new Set(recs.map((r) => variantComboFromRecord(r, fallback))); + if (allVariants.size > 1) { + for (const combo of allVariants) { + const subset = recs.filter((r) => variantComboFromRecord(r, fallback) === combo); + const substats = calcTwoPlayerStats(subset); + const variants = combo === "" ? [] : combo.split("|"); + const metaName = batchRatingGameLabel(metaUid, variants); + if (substats !== undefined) { + metaStats[metaName] = substats; + } + } + } + return metaStats; +} + +export function buildHMetaForGame( + recs: APGameRecord[], + metaUid: string, +): UserNumber | undefined { + const counts = new Map(); + for (const rec of recs) { + for (const prec of rec.header.players) { + if (prec.userid === undefined) { + continue; + } + counts.set(prec.userid, (counts.get(prec.userid) ?? 0) + 1); + } + } + if (counts.size === 0) { + return undefined; + } + return { user: metaUid, value: hIndexFromCounts(counts.values()) }; +} + +export function rateMetaGameVariants( + recs: APGameRecord[], + metaUid: string, + rater: ELOBasic, + ratingList: RatingListEntry[], + rawList: UserGameRating[], + fallback?: RecordGameIdFallback, +): void { + if (recs.length === 0) { + return; + } + const allVariants = new Set(recs.map((r) => variantComboFromRecord(r, fallback))); + if (allVariants.size === 0) { + return; + } + for (const combo of allVariants) { + console.log(`Rating game ${metaUid}, variant grouping ${combo}`); + const subset = recs.filter((r) => variantComboFromRecord(r, fallback) === combo); + const variants = combo === "" ? [] : combo.split("|"); + const metaName = batchRatingGameLabel(metaUid, variants); + + const results = rater.runProcessed(subset); + console.log( + `Elo rater:\nTotal records: ${results.recsReceived}, Num rated: ${results.recsRated}\n${ + results.warnings !== undefined ? results.warnings.join("\n") + "\n" : "" + }${results.errors !== undefined ? results.errors.join("\n") + "\n" : ""}`, + ); + for (const rating of results.ratings.values()) { + rating.gamename = metaUid; + const [, userid] = rating.userid.split("|"); + rating.userid = userid; + ratingList.push({ user: userid, game: metaName, rating }); + } + + console.log("Running Trueskill ratings"); + const ts = new Trueskill({ betaStart: 25 / 9 }); + const tsResults = ts.runProcessed(subset); + const tsRatings = new Map(tsResults.ratings) as Map; + if (ratingList.filter((r) => r.game === metaName).length !== tsRatings.size) { + const metaRatings = ratingList.filter((r) => r.game === metaName); + const elo = new Set(metaRatings.map((r) => r.user)); + const tsVals = new Set( + [...tsRatings.values()].map((r) => { + const [, u] = r.userid.split("|"); + return u; + }), + ); + const inElo = [...elo.values()].filter((u) => !tsVals.has(u)); + const inTS = [...tsVals.values()].filter((u) => !elo.has(u)); + throw new Error( + `The list of Elo ratings is not the same length as the list of Trueskill ratings.\nList of Elo ratings not in Trueskill: ${JSON.stringify(inElo, null, 2)}\nList of Trueskill ratings not in Elo: ${JSON.stringify(inTS, null, 2)}\nTrueskill ratings: ${JSON.stringify(tsRatings, replacer, 2)}`, + ); + } + console.log(`Final Trueskill ratings:\n${JSON.stringify([...tsRatings.values()])}`); + + console.log("Running Glicko2 ratings"); + const glicko = new Glicko2({ + ratingStart: GLICKO_RATING_START, + rdStart: GLICKO_RD_START, + }); + const oldest = new Date( + subset.map((r) => r.header["date-end"]).sort((a, b) => a.localeCompare(b))[0]!, + ); + const newest = new Date( + subset.map((r) => r.header["date-end"]).sort((a, b) => b.localeCompare(a))[0]!, + ); + console.log(`Oldest: ${oldest}, Newest: ${newest}`); + const oldestMs = oldest.getTime(); + const delta = newest.getTime() - oldestMs; + const period = GLICKO_PERIOD_MS; + const numPeriods = computeGlickoNumPeriods(delta, period); + console.log(`Number of periods: ${numPeriods}`); + const dated = subset.map((rec) => ({ + rec, + dateEndMs: new Date(rec.header["date-end"]).getTime(), + })); + const buckets = partitionByGlickoPeriod(dated, oldestMs, period, numPeriods); + let toDate = new Map(); + let ratedRecs = 0; + for (let p = 0; p < numPeriods; p++) { + glicko.knownRatings = new Map(toDate); + const periodRecs = buckets[p]!.map((d) => d.rec); + ratedRecs += periodRecs.length; + const glickoResults = glicko.runProcessed(periodRecs); + toDate = new Map(glickoResults.ratings as Map); + } + if (ratedRecs !== subset.length) { + throw new Error( + `The record subset had ${subset.length} records, but only ${ratedRecs} were handed to the rater.`, + ); + } + if (ratingList.filter((r) => r.game === metaName).length !== toDate.size) { + const metaRatings = ratingList.filter((r) => r.game === metaName); + const elo = new Set(metaRatings.map((r) => r.user)); + const glickoUsers = new Set( + [...toDate.values()].map((r) => { + const [, u] = r.userid.split("|"); + return u; + }), + ); + const inElo = [...elo.values()].filter((u) => !glickoUsers.has(u)); + const inGlicko = [...glickoUsers.values()].filter((u) => !elo.has(u)); + throw new Error( + `The list of Elo ratings is not the same length as the list of Glicko ratings.\nList of Elo ratings not in Glicko: ${JSON.stringify(inElo, null, 2)}\nList of Glicko ratings not in Elo: ${JSON.stringify(inGlicko, null, 2)}\nGlicko ratings: ${JSON.stringify(toDate, replacer, 2)}`, + ); + } + console.log(`Final glicko rating results: ${JSON.stringify(toDate, replacer)}`); + + for (const userStr of toDate.keys()) { + const [, user] = userStr.split("|"); + const elo = ratingList.find((r) => r.user === user && r.game === metaName)?.rating; + if (elo === undefined) { + throw new Error(`Could not find a matching Elo rating for ${user}.`); + } + const ts = tsRatings.get(userStr); + if (ts === undefined) { + throw new Error(`Could not find a matching Trueskill rating for ${user}.`); + } + const glickoRating = toDate.get(userStr)!; + if (elo.recCount !== glickoRating.recCount) { + throw new Error("Rated recCounts do not match."); + } + if (elo.recCount !== ts.recCount) { + throw new Error( + `Rated recCounts do not match for user ${user}:\nElo: ${elo.recCount}\nTrueskill: ${ts.recCount}`, + ); + } + rawList.push({ + user, + game: metaName, + rating: Math.round(elo.rating), + wld: [elo.wins, elo.losses, elo.draws], + glicko: toGlickoStats( + glickoRating.rating, + glickoRating.rd, + glickoRating.volatility, + glickoRating.recCount, + ), + trueskill: { mu: ts.rating, sigma: ts.sigma }, + }); + } + } +} diff --git a/crons/src/functions/summarizeScan.test.ts b/crons/src/functions/summarizeScan.test.ts new file mode 100644 index 00000000..559890bc --- /dev/null +++ b/crons/src/functions/summarizeScan.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import type { APGameRecord } from "@abstractplay/recranks"; +import { + accumulateRivalryPair, + finalizeRivalryPairs, +} from "./summarizeHelpers.js"; +import { + buildPastDisplayNamesList, + buildPlayerStats, + createSummarizeScanState, + scanRecord, +} from "./summarizeScan.js"; + +const CHESS_GAMEID = "f47ac10b-58cc-4372-a567-0e02b2c3d479#chess:"; +const CHESS_GAMEID_2 = "b47ac10b-58cc-4372-a567-0e02b2c3d481#chess:"; +const GO_GAMEID = "a47ac10b-58cc-4372-a567-0e02b2c3d480#go:"; + +function minimalRec(opts: { + gameid: string; + gameName: string; + dateEnd: string; + p1: string; + p2: string; + p1Name?: string; + p2Name?: string; +}): APGameRecord { + return { + header: { + game: { name: opts.gameName }, + site: { name: "Abstract Play", gameid: opts.gameid }, + "date-start": opts.dateEnd, + "date-end": opts.dateEnd, + "date-generated": opts.dateEnd, + players: [ + { name: opts.p1Name ?? "A", userid: opts.p1, result: 1 }, + { name: opts.p2Name ?? "B", userid: opts.p2, result: 0 }, + ], + }, + moves: [["e4", "e5"], ["d4", "d5"]], + } as APGameRecord; +} + +describe("summarizeScan", () => { + it("accumulates player stats in one pass without storing full record lists", () => { + const state = createSummarizeScanState(); + const gameInfo = new Map(); + scanRecord(state, minimalRec({ gameid: CHESS_GAMEID, gameName: "Chess", dateEnd: "2024-01-01T00:00:00Z", p1: "alice", p2: "bob" }), gameInfo); + scanRecord(state, minimalRec({ gameid: GO_GAMEID, gameName: "Go", dateEnd: "2024-01-02T00:00:00Z", p1: "alice", p2: "carol" }), gameInfo); + + expect(state.numGames).toBe(2); + expect(state.playerIDs).toEqual(new Set(["alice", "bob", "carol"])); + + const stats = buildPlayerStats(state); + const alice = stats.allPlays.find((r) => r.user === "alice"); + expect(alice?.value).toBe(2); + expect(stats.eclectic.find((r) => r.user === "alice")?.value).toBe(2); + expect(stats.social.find((r) => r.user === "alice")?.value).toBe(2); + }); + + it("matches batch rivalry counting via incremental accumulator", () => { + const counts = new Map(); + const recs = [ + minimalRec({ gameid: CHESS_GAMEID, gameName: "Chess", dateEnd: "2024-01-01T00:00:00Z", p1: "alice", p2: "bob" }), + minimalRec({ gameid: CHESS_GAMEID_2, gameName: "Chess", dateEnd: "2024-01-02T00:00:00Z", p1: "alice", p2: "bob" }), + ]; + for (const rec of recs) { + accumulateRivalryPair(counts, rec); + } + const pairs = finalizeRivalryPairs(counts, 2); + expect(pairs).toEqual([{ userA: "alice", userB: "bob", n: 2 }]); + }); + + it("buildPastDisplayNamesList excludes current USERS name", () => { + const state = createSummarizeScanState(); + const gameInfo = new Map(); + scanRecord( + state, + minimalRec({ + gameid: CHESS_GAMEID, + gameName: "Chess", + dateEnd: "2024-01-01T00:00:00Z", + p1: "alice", + p2: "bob", + p1Name: "Alice Old", + }), + gameInfo, + ); + scanRecord( + state, + minimalRec({ + gameid: CHESS_GAMEID_2, + gameName: "Chess", + dateEnd: "2024-01-02T00:00:00Z", + p1: "alice", + p2: "bob", + p1Name: "Alice Current", + }), + gameInfo, + ); + const current = new Map([ + ["alice", "Alice Current"], + ["bob", "B"], + ]); + const list = buildPastDisplayNamesList(state.pastNamesByUser, current); + expect(list).toEqual([{ user: "alice", names: ["Alice Old"] }]); + }); +}); diff --git a/crons/src/functions/summarizeScan.ts b/crons/src/functions/summarizeScan.ts new file mode 100644 index 00000000..90d75f2b --- /dev/null +++ b/crons/src/functions/summarizeScan.ts @@ -0,0 +1,469 @@ +import type { APGameRecord } from "@abstractplay/recranks"; +import type { GameNumber } from "types/stats/GameNumber.js"; +import type { GameNumList } from "types/stats/GameNumList.js"; +import type { MetaPieStats } from "types/stats/MetaPieStats.js"; +import type { MetaPlayerCountMix } from "types/stats/MetaPlayerCountMix.js"; +import type { UserNumList } from "types/stats/UserNumList.js"; +import type { UserNumber } from "types/stats/UserNumber.js"; +import { + accumulateRivalryPair, + buildPlayerTimeoutHistograms, + computeReturningPlayersPerWeek, + computeTimeoutHistogramRates, + findTimeoutPlayerSeat, + gameSupportsMultiPlayerCount, + gameSupportsPie, + maxOf, + recordHasAbandoned, + recordHasTimeout, + recordMoveSlotCount, + recordRoundCount, + recordPlayerTimeout, + recordWasPied, + timeoutStatsFromAccumulator, + hIndexFromCounts, + metaGameFromRecord, + type RecordGameIdFallback, + type HoursPerGameInput, + type PlayerTimeoutAccumulator, + type RivalryPairResult, +} from "./summarizeHelpers.js"; + +export type GameInfoFlags = { + name: string; + flags?: string[]; + playercounts: number[]; +}; + +export type SummarizeScanState = { + numGames: number; + playerIDs: Set; + oldest?: string; + newest?: string; + casualGames: number; + eventGames: number; + playerTimeoutAcc: Map; + siteEndFailures: number[]; + siteClockTimeouts: number[]; + siteAbandonments: number[]; + metaPlayCount: Map; + metaPlayUsers: Map>; + playerAllPlays: Map; + playerEclecticGames: Map>; + playerSocialOpps: Map>; + playerGameCounts: Map>; + playerOppCounts: Map>; + rivalryCounts: Map; + histList: { game: string; bucket: number }[]; + histListPlayers: { user: string; bucket: number }[]; + completedList: { user: string; time: number }[]; + earliestMs?: number; + pieByGame: Map; + playerCountMixByGame: Map>; + hoursPerGames: HoursPerGameInput[]; + recentCompleterIDs: Set; + pastNamesByUser: Map>; +}; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; +const ACTIVE_GEO_DAYS = 30; + +export function createSummarizeScanState(): SummarizeScanState { + return { + numGames: 0, + playerIDs: new Set(), + casualGames: 0, + eventGames: 0, + playerTimeoutAcc: new Map(), + siteEndFailures: [], + siteClockTimeouts: [], + siteAbandonments: [], + metaPlayCount: new Map(), + metaPlayUsers: new Map(), + playerAllPlays: new Map(), + playerEclecticGames: new Map(), + playerSocialOpps: new Map(), + playerGameCounts: new Map(), + playerOppCounts: new Map(), + rivalryCounts: new Map(), + histList: [], + histListPlayers: [], + completedList: [], + pieByGame: new Map(), + playerCountMixByGame: new Map(), + hoursPerGames: [], + recentCompleterIDs: new Set(), + pastNamesByUser: new Map(), + }; +} + +export function buildPastDisplayNamesList( + pastNamesByUser: Map>, + currentDisplayNames: Map, +): { user: string; names: string[] }[] { + const result: { user: string; names: string[] }[] = []; + for (const [user, names] of pastNamesByUser) { + const current = currentDisplayNames.get(user)?.trim(); + const filtered = [...names] + .map((n) => n.trim()) + .filter((n) => n.length > 0 && n !== current); + const unique = [...new Set(filtered)].sort((a, b) => a.localeCompare(b)); + if (unique.length > 0) { + result.push({ user, names: unique }); + } + } + result.sort((a, b) => a.user.localeCompare(b.user)); + return result; +} + +function incrementMapCount(map: Map, key: string, delta = 1): void { + map.set(key, (map.get(key) ?? 0) + delta); +} + +function nestedIncrement( + outer: Map>, + outerKey: string, + innerKey: string, +): void { + let inner = outer.get(outerKey); + if (inner === undefined) { + inner = new Map(); + outer.set(outerKey, inner); + } + inner.set(innerKey, (inner.get(innerKey) ?? 0) + 1); +} + +export function scanRecord( + state: SummarizeScanState, + rec: APGameRecord, + gameInfoByUid: Map, + fallback?: RecordGameIdFallback, +): void { + state.numGames++; + const metaUid = metaGameFromRecord(rec, fallback); + const dateEnd = rec.header["date-end"]; + const completedMs = new Date(dateEnd).getTime(); + + if (state.oldest === undefined || dateEnd < state.oldest) { + state.oldest = dateEnd; + } + if (state.newest === undefined || dateEnd > state.newest) { + state.newest = dateEnd; + } + if (state.earliestMs === undefined || completedMs < state.earliestMs) { + state.earliestMs = completedMs; + } + + incrementMapCount(state.metaPlayCount, metaUid); + let metaUsers = state.metaPlayUsers.get(metaUid); + if (metaUsers === undefined) { + metaUsers = new Set(); + state.metaPlayUsers.set(metaUid, metaUsers); + } + + const playerIdsInRec: string[] = []; + for (const p of rec.header.players) { + if (p.userid !== undefined && typeof p.name === "string") { + const embeddedName = p.name.trim(); + if (embeddedName.length > 0) { + let nameSet = state.pastNamesByUser.get(p.userid); + if (nameSet === undefined) { + nameSet = new Set(); + state.pastNamesByUser.set(p.userid, nameSet); + } + nameSet.add(embeddedName); + } + } + if (p.userid === undefined) { + continue; + } + const user = p.userid; + state.playerIDs.add(user); + playerIdsInRec.push(user); + metaUsers.add(user); + incrementMapCount(state.playerAllPlays, user); + + let eclectic = state.playerEclecticGames.get(user); + if (eclectic === undefined) { + eclectic = new Set(); + state.playerEclecticGames.set(user, eclectic); + } + eclectic.add(metaUid); + nestedIncrement(state.playerGameCounts, user, metaUid); + } + + for (const user of playerIdsInRec) { + let opps = state.playerSocialOpps.get(user); + if (opps === undefined) { + opps = new Set(); + state.playerSocialOpps.set(user, opps); + } + for (const other of playerIdsInRec) { + if (other !== user) { + opps.add(other); + nestedIncrement(state.playerOppCounts, user, other); + } + } + } + + if (rec.header.event !== undefined && rec.header.event !== "") { + state.eventGames++; + } else { + state.casualGames++; + } + + if (recordHasAbandoned(rec.moves)) { + state.siteEndFailures.push(completedMs); + state.siteAbandonments.push(completedMs); + } else if (recordHasTimeout(rec.moves)) { + state.siteEndFailures.push(completedMs); + state.siteClockTimeouts.push(completedMs); + const seatIdx = findTimeoutPlayerSeat(rec.moves, rec.header.players.length); + if (seatIdx !== undefined) { + const p = rec.header.players[seatIdx]; + if (p.userid !== undefined) { + recordPlayerTimeout(state.playerTimeoutAcc, p.userid, completedMs); + } + } + } + + accumulateRivalryPair(state.rivalryCounts, rec); + + const found = gameInfoByUid.get(metaUid); + if (found !== undefined) { + if (gameSupportsPie(found.flags)) { + const acc = state.pieByGame.get(metaUid) ?? { n: 0, pied: 0 }; + acc.n++; + if (recordWasPied(rec.header)) { + acc.pied++; + } + state.pieByGame.set(metaUid, acc); + } + if (gameSupportsMultiPlayerCount(found.playercounts)) { + const key = String(rec.header.players.length); + let byCount = state.playerCountMixByGame.get(metaUid); + if (byCount === undefined) { + byCount = new Map(); + state.playerCountMixByGame.set(metaUid, byCount); + } + byCount.set(key, (byCount.get(key) ?? 0) + 1); + } + } + + const activeGeoCutoffMs = Date.now() - ACTIVE_GEO_DAYS * MS_PER_DAY; + if (completedMs >= activeGeoCutoffMs) { + for (const user of playerIdsInRec) { + state.recentCompleterIDs.add(user); + } + } + + const earliest = state.earliestMs!; + const daysAgo = (completedMs - earliest) / MS_PER_DAY; + const bucket = Math.floor(daysAgo / 7); + state.histList.push({ game: metaUid, bucket }); + for (const user of playerIdsInRec) { + state.histListPlayers.push({ user, bucket }); + state.completedList.push({ user, time: completedMs }); + } + + if ( + !recordHasTimeout(rec.moves) && + !recordHasAbandoned(rec.moves) && + recordRoundCount(rec) >= 2 && + rec.header["date-start"] !== undefined + ) { + const started = new Date(rec.header["date-start"]).getTime(); + const moveSlots = recordMoveSlotCount(rec); + if (moveSlots > 0) { + state.hoursPerGames.push({ dateStartMs: started, dateEndMs: completedMs, moveSlots }); + } + } +} + +export function buildPlayStats(state: SummarizeScanState): { + numPlays: GameNumber[]; + playWidth: GameNumber[]; +} { + const numPlays: GameNumber[] = []; + const playWidth: GameNumber[] = []; + for (const [game, count] of state.metaPlayCount.entries()) { + numPlays.push({ game, value: count }); + playWidth.push({ game, value: state.metaPlayUsers.get(game)?.size ?? 0 }); + } + return { numPlays, playWidth }; +} + +export function buildPlayerStats(state: SummarizeScanState): { + allPlays: UserNumber[]; + eclectic: UserNumber[]; + social: UserNumber[]; + h: UserNumber[]; + hOpp: UserNumber[]; +} { + const allPlays: UserNumber[] = []; + const eclectic: UserNumber[] = []; + const social: UserNumber[] = []; + const h: UserNumber[] = []; + const hOpp: UserNumber[] = []; + + for (const [user, count] of state.playerAllPlays.entries()) { + allPlays.push({ user, value: count }); + eclectic.push({ user, value: state.playerEclecticGames.get(user)?.size ?? 0 }); + social.push({ user, value: state.playerSocialOpps.get(user)?.size ?? 0 }); + + const gameCounts = state.playerGameCounts.get(user); + h.push({ + user, + value: gameCounts === undefined ? 0 : hIndexFromCounts(gameCounts.values()), + }); + + const oppCounts = state.playerOppCounts.get(user); + hOpp.push({ + user, + value: oppCounts === undefined ? 0 : hIndexFromCounts(oppCounts.values()), + }); + } + + return { allPlays, eclectic, social, h, hOpp }; +} + +export function buildPieRates(state: SummarizeScanState): MetaPieStats[] { + const pieRates: MetaPieStats[] = []; + for (const [game, acc] of state.pieByGame.entries()) { + pieRates.push({ + game, + n: acc.n, + pied: acc.pied, + rate: acc.n > 0 ? acc.pied / acc.n : 0, + }); + } + pieRates.sort((a, b) => a.game.localeCompare(b.game)); + return pieRates; +} + +export function buildPlayerCountMix(state: SummarizeScanState): MetaPlayerCountMix[] { + const mix: MetaPlayerCountMix[] = []; + for (const [game, byCount] of state.playerCountMixByGame.entries()) { + const counts: { [playerCount: string]: number } = {}; + for (const [key, value] of byCount.entries()) { + counts[key] = value; + } + mix.push({ game, byCount: counts }); + } + mix.sort((a, b) => a.game.localeCompare(b.game)); + return mix; +} + +export function buildSiteHistograms(state: SummarizeScanState): { + histAll: number[]; + histAllPlayers: number[]; + histTimeouts: number[]; + histAbandoned: number[]; + histMeta: GameNumList[]; + histPlayers: UserNumList[]; + firstTimers: number[]; + returningPlayers: number[]; + recent: GameNumber[]; + maxBucket: number; + histPlayerTimeouts: UserNumList[]; + timeoutStats: ReturnType; +} { + const earliest = state.earliestMs ?? 0; + let maxBucket = maxOf(state.histList.map((x) => x.bucket)); + + const histAll: number[] = []; + const histAllPlayers: number[] = []; + for (let i = 0; i <= maxBucket; i++) { + histAll.push(state.histList.filter((x) => x.bucket === i).length); + const users = new Set(); + for (const rec of state.histListPlayers.filter((x) => x.bucket === i)) { + users.add(rec.user); + } + histAllPlayers.push(users.size); + } + + const histTimeoutBuckets: number[] = []; + for (const t of state.siteClockTimeouts) { + const daysAgo = (t - earliest) / MS_PER_DAY; + histTimeoutBuckets.push(Math.floor(daysAgo / 7)); + } + const histTimeoutCounts: number[] = []; + for (let i = 0; i <= maxOf(histTimeoutBuckets); i++) { + histTimeoutCounts.push(histTimeoutBuckets.filter((x) => x === i).length); + } + const histTimeouts = computeTimeoutHistogramRates(histTimeoutCounts, histAll); + + const histAbandonedBuckets: number[] = []; + for (const t of state.siteAbandonments) { + const daysAgo = (t - earliest) / MS_PER_DAY; + histAbandonedBuckets.push(Math.floor(daysAgo / 7)); + } + const histAbandonedCounts: number[] = []; + for (let i = 0; i <= maxOf(histAbandonedBuckets); i++) { + histAbandonedCounts.push(histAbandonedBuckets.filter((x) => x === i).length); + } + const histAbandoned = computeTimeoutHistogramRates(histAbandonedCounts, histAll); + + const histMeta: GameNumList[] = []; + const recent: GameNumber[] = []; + const metaNames = new Set(state.histList.map((x) => x.game)); + for (const meta of metaNames) { + const subset = state.histList.filter((x) => x.game === meta); + const metaMax = maxOf(subset.map((x) => x.bucket)); + const lst: number[] = []; + for (let i = 0; i <= metaMax; i++) { + lst.push(subset.filter((x) => x.bucket === i).length); + } + histMeta.push({ game: meta, value: [...lst] }); + const slice = lst.slice(-4); + recent.push({ game: meta, value: slice.reduce((prev, curr) => prev + curr, 0) }); + } + + const histPlayers: UserNumList[] = []; + const userIds = new Set(state.histListPlayers.map((x) => x.user)); + for (const userid of userIds) { + const subset = state.histListPlayers.filter((x) => x.user === userid); + const userMax = maxOf(subset.map((x) => x.bucket)); + const lst: number[] = []; + for (let i = 0; i <= userMax; i++) { + lst.push(subset.filter((x) => x.bucket === i).length); + } + histPlayers.push({ user: userid, value: [...lst] }); + } + + const timeoutStats = timeoutStatsFromAccumulator(state.playerTimeoutAcc); + const histPlayerTimeouts = buildPlayerTimeoutHistograms( + state.playerTimeoutAcc, + userIds, + earliest, + ); + + const buckets: number[] = []; + for (const userid of userIds) { + const times = state.completedList.filter((x) => x.user === userid).map((x) => x.time); + const localEarliest = Math.min(...times); + const daysAgo = (localEarliest - earliest) / MS_PER_DAY; + buckets.push(Math.floor(daysAgo / 7)); + } + const firstTimers: number[] = []; + maxBucket = maxOf(buckets); + for (let i = 0; i <= maxBucket; i++) { + firstTimers.push(buckets.filter((x) => x === i).length); + } + const returningPlayers = computeReturningPlayersPerWeek(state.completedList, earliest, maxBucket); + + return { + histAll, + histAllPlayers, + histTimeouts, + histAbandoned, + histMeta, + histPlayers, + firstTimers, + returningPlayers, + recent, + maxBucket, + histPlayerTimeouts, + timeoutStats, + }; +} diff --git a/crons/src/functions/summarizeSolo.test.ts b/crons/src/functions/summarizeSolo.test.ts new file mode 100644 index 00000000..1b3ddb1a --- /dev/null +++ b/crons/src/functions/summarizeSolo.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import type { APGameRecord } from "@abstractplay/recranks"; +import { + accumulateSoloRecord, + buildSoloMetaStats, + buildSoloSeedBoards, + createSoloSummarizeState, +} from "./summarizeSolo.js"; + +const SEED = "20260819-graded-fixture"; +const META_UID = "solo-puzzle"; +const VARIANT_KEY = `${META_UID} (standard)`; + +function makeGradedSoloRecord(overrides: { + gameid?: string; + score?: number; + grade?: string; + dateEnd?: string; +} = {}): APGameRecord { + return { + header: { + game: { name: "Solo Puzzle", variants: ["standard"] }, + site: { name: "Abstract Play", gameid: overrides.gameid ?? "solo-graded-1" }, + "date-start": "2026-08-19T14:00:00.000Z", + "date-end": overrides.dateEnd ?? "2026-08-19T14:30:00.000Z", + "date-generated": "2026-08-19T14:30:01.000Z", + "outcome-type": "graded", + "score-direction": "higher", + "score-label": "points", + "challenge-seed": SEED, + unrated: true, + players: [{ + name: "alice", + userid: "user-alice", + score: overrides.score ?? 73, + grade: overrides.grade ?? "good", + result: 1, + }], + }, + moves: [ + ["score"], + ["score"], + ["finish"], + ], + }; +} + +describe("summarizeSolo", () => { + it("aggregates three attempts from one user into meta stats and seed board", () => { + const state = createSoloSummarizeState(); + const attempts = [ + makeGradedSoloRecord({ gameid: "solo-1", score: 60, grade: "ok", dateEnd: "2026-08-19T14:10:00.000Z" }), + makeGradedSoloRecord({ gameid: "solo-2", score: 73, grade: "good", dateEnd: "2026-08-19T14:20:00.000Z" }), + makeGradedSoloRecord({ gameid: "solo-3", score: 90, grade: "excellent", dateEnd: "2026-08-19T14:30:00.000Z" }), + ]; + for (const rec of attempts) { + accumulateSoloRecord(state, rec, { + resolveMetaUidFromDisplayName: () => META_UID, + }); + } + + const metaStats = buildSoloMetaStats(state); + const variantStats = metaStats[VARIANT_KEY]; + expect(variantStats).toBeDefined(); + expect(variantStats!.attempts).toBe(3); + expect(variantStats!.uniquePlayers).toBe(1); + expect(variantStats!.repeatAttemptRate).toBeCloseTo(2 / 3); + expect(variantStats!.scoreMedianAllAttempts).toBe(73); + expect(variantStats!.scoreMedianBestPerUser).toBe(90); + expect(variantStats!.outcomeTypes).toEqual({ graded: 3 }); + expect(variantStats!.gradeHistogramBestPerUser).toEqual({ excellent: 1 }); + + const boards = buildSoloSeedBoards(state); + expect(boards).toHaveLength(1); + const board = boards[0]!; + expect(board.challengeSeed).toBe(SEED); + expect(board.attempts).toBe(3); + expect(board.uniquePlayers).toBe(1); + expect(board.rows).toHaveLength(1); + expect(board.rows[0]).toMatchObject({ + userid: "user-alice", + score: 90, + grade: "excellent", + attempts: 3, + }); + expect(board.scoreMedianAllAttempts).toBe(73); + expect(board.scoreMedianBestPerUser).toBe(90); + }); + + it("ignores multiplayer records", () => { + const state = createSoloSummarizeState(); + const rec: APGameRecord = { + header: { + game: { name: "Chess" }, + site: { name: "Abstract Play", gameid: "chess-1" }, + "date-end": "2026-08-19T14:30:00.000Z", + players: [ + { name: "a", userid: "user-a", result: 1 }, + { name: "b", userid: "user-b", result: 0 }, + ], + }, + moves: [["e4"], ["e5"]], + }; + accumulateSoloRecord(state, rec); + expect(buildSoloMetaStats(state)).toEqual({}); + expect(buildSoloSeedBoards(state)).toEqual([]); + }); +}); diff --git a/crons/src/functions/summarizeSolo.ts b/crons/src/functions/summarizeSolo.ts new file mode 100644 index 00000000..92069a97 --- /dev/null +++ b/crons/src/functions/summarizeSolo.ts @@ -0,0 +1,296 @@ +import type { APGameRecord } from "@abstractplay/recranks"; +import { batchRatingGameLabel } from "../lib/batchRatings.js"; +import type { + ScoreDirection, + SoloMetaStats, + SoloOutcomeType, + SoloSeedBoard, + SoloSeedBoardRow, +} from "types/stats/SoloStats.js"; +import { + medianOf, + metaGameFromRecord, + percentileOf, + recordRoundCount, + variantUidsFromRecord, + type RecordGameIdFallback, +} from "./summarizeHelpers.js"; + +type SoloPlayer = { + score?: number; + grade?: string; + passed?: boolean; + result?: number; + userid?: string; + name: string; +}; + +type SoloHeader = APGameRecord["header"] & { + "outcome-type"?: SoloOutcomeType; + "score-direction"?: ScoreDirection; + "challenge-seed"?: string; +}; + +export type SoloVariantBucket = { + metaUid: string; + variants: string[]; + records: APGameRecord[]; +}; + +export type SoloSeedBucket = { + variantKey: string; + metaUid: string; + variants: string[]; + seed: string; + records: APGameRecord[]; +}; + +export type SoloSummarizeState = { + byVariant: Map; + bySeed: Map; +}; + +const soloHeader = (rec: APGameRecord): SoloHeader => rec.header as SoloHeader; + +export const isSoloRecord = (rec: APGameRecord): boolean => rec.header.players.length === 1; + +const playerScore = (rec: APGameRecord): number => { + const player = rec.header.players[0] as SoloPlayer; + return player.score ?? player.result ?? 0; +}; + +const scoreDirection = (rec: APGameRecord): ScoreDirection => + soloHeader(rec)["score-direction"] ?? "higher"; + +/** True when `candidate` ranks better than `incumbent` for this bucket's score direction. */ +export const soloAttemptIsBetter = (candidate: APGameRecord, incumbent: APGameRecord): boolean => { + const dir = scoreDirection(candidate); + const next = playerScore(candidate); + const prev = playerScore(incumbent); + if (dir === "higher") { + if (next !== prev) { + return next > prev; + } + } else if (next !== prev) { + return next < prev; + } + const nextMoves = recordRoundCount(candidate); + const prevMoves = recordRoundCount(incumbent); + if (nextMoves !== prevMoves) { + return nextMoves < prevMoves; + } + return candidate.header["date-end"] < incumbent.header["date-end"]; +}; + +export function createSoloSummarizeState(): SoloSummarizeState { + return { + byVariant: new Map(), + bySeed: new Map(), + }; +} + +export function accumulateSoloRecord( + state: SoloSummarizeState, + rec: APGameRecord, + fallback?: RecordGameIdFallback, +): void { + if (!isSoloRecord(rec)) { + return; + } + const player = rec.header.players[0]; + if (player.userid === undefined || player.userid === "") { + return; + } + + const metaUid = metaGameFromRecord(rec, fallback); + const variants = variantUidsFromRecord(rec, fallback); + const variantKey = batchRatingGameLabel(metaUid, variants); + + let variantBucket = state.byVariant.get(variantKey); + if (variantBucket === undefined) { + variantBucket = { metaUid, variants, records: [] }; + state.byVariant.set(variantKey, variantBucket); + } + variantBucket.records.push(rec); + + const seed = soloHeader(rec)["challenge-seed"]; + if (typeof seed === "string" && seed.length > 0) { + const seedKey = `${variantKey}\t${seed}`; + let seedBucket = state.bySeed.get(seedKey); + if (seedBucket === undefined) { + seedBucket = { variantKey, metaUid, variants, seed, records: [] }; + state.bySeed.set(seedKey, seedBucket); + } + seedBucket.records.push(rec); + } +} + +type BestPerUser = Map; + +const bestPerUser = (records: APGameRecord[]): BestPerUser => { + const byUser: BestPerUser = new Map(); + for (const rec of records) { + const userid = rec.header.players[0].userid; + if (userid === undefined || userid === "") { + continue; + } + const existing = byUser.get(userid); + if (existing === undefined) { + byUser.set(userid, { best: rec, attempts: 1 }); + } else { + existing.attempts += 1; + if (soloAttemptIsBetter(rec, existing.best)) { + existing.best = rec; + } + } + } + return byUser; +}; + +const outcomeTypeCounts = (records: APGameRecord[]): Partial> => { + const counts: Partial> = {}; + for (const rec of records) { + const outcomeType = soloHeader(rec)["outcome-type"]; + if (outcomeType === undefined) { + continue; + } + counts[outcomeType] = (counts[outcomeType] ?? 0) + 1; + } + return counts; +}; + +const passRateAllAttempts = (records: APGameRecord[]): number | undefined => { + const binary = records.filter((rec) => soloHeader(rec)["outcome-type"] === "binary"); + if (binary.length === 0) { + return undefined; + } + const passed = binary.filter((rec) => (rec.header.players[0] as SoloPlayer).passed === true); + return passed.length / binary.length; +}; + +const passRateBestPerUser = (byUser: BestPerUser): number | undefined => { + const binary = [...byUser.values()].filter( + (entry) => soloHeader(entry.best)["outcome-type"] === "binary", + ); + if (binary.length === 0) { + return undefined; + } + const passed = binary.filter((entry) => (entry.best.header.players[0] as SoloPlayer).passed === true); + return passed.length / binary.length; +}; + +const gradeHistogramBestPerUser = (byUser: BestPerUser): Record | undefined => { + const grades: Record = {}; + let hasGrade = false; + for (const entry of byUser.values()) { + const grade = (entry.best.header.players[0] as SoloPlayer).grade; + if (grade === undefined || grade === "") { + continue; + } + hasGrade = true; + grades[grade] = (grades[grade] ?? 0) + 1; + } + return hasGrade ? grades : undefined; +}; + +const buildSeedBoardRows = (byUser: BestPerUser, sample: APGameRecord): SoloSeedBoardRow[] => { + const dir = scoreDirection(sample); + const rows: SoloSeedBoardRow[] = []; + for (const [userid, entry] of byUser.entries()) { + const player = entry.best.header.players[0] as SoloPlayer & (typeof entry.best.header.players)[0]; + rows.push({ + userid, + name: player.name, + score: playerScore(entry.best), + grade: player.grade, + passed: player.passed, + dateEnd: entry.best.header["date-end"], + attempts: entry.attempts, + }); + } + rows.sort((a, b) => { + if (dir === "higher") { + if (a.score !== b.score) { + return b.score - a.score; + } + } else if (a.score !== b.score) { + return a.score - b.score; + } + return a.dateEnd.localeCompare(b.dateEnd); + }); + return rows; +}; + +const scoreStats = (records: APGameRecord[], byUser: BestPerUser) => { + const allScores = records.map(playerScore); + const bestScores = [...byUser.values()].map((entry) => playerScore(entry.best)); + return { + scoreMedianAllAttempts: medianOf(allScores), + scoreMedianBestPerUser: medianOf(bestScores), + scoreP90BestPerUser: percentileOf(bestScores, 90), + }; +}; + +export function buildSoloMetaStats(state: SoloSummarizeState): Record { + const result: Record = {}; + for (const [variantKey, bucket] of state.byVariant.entries()) { + const { metaUid, variants, records } = bucket; + if (records.length === 0) { + continue; + } + const byUser = bestPerUser(records); + const uniquePlayers = byUser.size; + const attempts = records.length; + const scores = scoreStats(records, byUser); + const moveCounts = records.map(recordRoundCount); + result[variantKey] = { + game: variantKey, + metaUid, + variants, + attempts, + uniquePlayers, + repeatAttemptRate: attempts > 0 ? (attempts - uniquePlayers) / attempts : 0, + outcomeTypes: outcomeTypeCounts(records), + ...scores, + passRateAllAttempts: passRateAllAttempts(records), + passRateBestPerUser: passRateBestPerUser(byUser), + gradeHistogramBestPerUser: gradeHistogramBestPerUser(byUser), + moveCountMedian: medianOf(moveCounts), + }; + } + return result; +} + +export function buildSoloSeedBoards(state: SoloSummarizeState): SoloSeedBoard[] { + const boards: SoloSeedBoard[] = []; + for (const bucket of state.bySeed.values()) { + const { variantKey, metaUid, variants, seed, records } = bucket; + if (records.length === 0) { + continue; + } + const byUser = bestPerUser(records); + const sample = records[0]!; + const scores = scoreStats(records, byUser); + boards.push({ + game: variantKey, + metaUid, + variants, + challengeSeed: seed, + scoreDirection: scoreDirection(sample), + outcomeType: soloHeader(sample)["outcome-type"], + attempts: records.length, + uniquePlayers: byUser.size, + scoreMedianAllAttempts: scores.scoreMedianAllAttempts, + scoreMedianBestPerUser: scores.scoreMedianBestPerUser, + rows: buildSeedBoardRows(byUser, sample), + }); + } + boards.sort((a, b) => { + const byGame = a.game.localeCompare(b.game); + if (byGame !== 0) { + return byGame; + } + return a.challengeSeed.localeCompare(b.challengeSeed); + }); + return boards; +} diff --git a/crons/src/functions/thumbnailInterop.test.ts b/crons/src/functions/thumbnailInterop.test.ts new file mode 100644 index 00000000..ac275f7d --- /dev/null +++ b/crons/src/functions/thumbnailInterop.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "vitest"; +import { gameinfo, GameFactory } from "@abstractplay/gameslib"; +import { addPrefix, render } from "@abstractplay/renderer"; + +describe("thumbnail pipeline package interop", () => { + it("loads gameslib via ESM import", () => { + expect(gameinfo).toBeTruthy(); + expect(typeof GameFactory).toBe("function"); + }); + + it("exposes renderer addPrefix and render", () => { + expect(typeof addPrefix).toBe("function"); + expect(typeof render).toBe("function"); + }); +}); diff --git a/crons/src/functions/thumbnails-verify.ts b/crons/src/functions/thumbnails-verify.ts new file mode 100644 index 00000000..7bee6962 --- /dev/null +++ b/crons/src/functions/thumbnails-verify.ts @@ -0,0 +1,61 @@ +import { HeadObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { gameinfo, type APGamesInformation } from "@abstractplay/gameslib"; +import type { Handler } from "aws-lambda"; +import { THUMB_BUCKET, THUMBNAIL_BROKEN_METAS } from "../utils/thumbnailConfig.js"; +import { + findThumbnailFreshnessMismatches, + type ObjectHead, +} from "../utils/thumbnailFreshness.js"; + +const s3 = new S3Client({ region: "us-east-1" }); + +async function headObject(bucket: string, key: string): Promise { + try { + const response = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); + if (!response.LastModified) { + return null; + } + return { key, lastModified: response.LastModified }; + } catch (err) { + const code = (err as { name?: string }).name; + if (code === "NotFound" || code === "NoSuchKey") { + return null; + } + throw err; + } +} + +export const handler: Handler = async () => { + const metas = ([...gameinfo.values()] as APGamesInformation[]) + .filter((rec) => !rec.flags.includes("experimental")) + .map((rec) => rec.uid); + + const heads = new Map(); + for (const meta of metas) { + if (THUMBNAIL_BROKEN_METAS.includes(meta)) { + continue; + } + for (const suffix of [".json", "-light.svg"]) { + const key = `${meta}${suffix}`; + const head = await headObject(THUMB_BUCKET, key); + if (head) { + heads.set(key, head); + } + } + } + + const mismatches = findThumbnailFreshnessMismatches(metas, heads, THUMBNAIL_BROKEN_METAS); + if (mismatches.length === 0) { + console.log(`Thumbnail freshness OK for ${metas.length} production metas`); + return { ok: true, checked: metas.length, mismatches: [] }; + } + + console.error( + `Thumbnail SVG freshness check failed for ${mismatches.length} meta(s):\n` + + JSON.stringify(mismatches, null, 2), + ); + const summary = mismatches + .map((m) => `${m.meta} (${m.reason}: json=${m.jsonLastModified}, svg=${m.svgLastModified ?? "missing"})`) + .join("; "); + throw new Error(`Thumbnail SVG freshness check failed: ${summary}`); +}; diff --git a/crons/src/functions/thumbnails.ts b/crons/src/functions/thumbnails.ts new file mode 100644 index 00000000..ea932c3c --- /dev/null +++ b/crons/src/functions/thumbnails.ts @@ -0,0 +1,249 @@ +import { S3Client, GetObjectCommand, ListObjectsV2Command, PutObjectCommand, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { GameFactory, addResource, gameinfo, type APGamesInformation } from "@abstractplay/gameslib"; +import enApgames from "@abstractplay/gameslib/locales/en/apgames.json"; +import enApresults from "@abstractplay/gameslib/locales/en/apresults.json"; +import { gunzipSync, strFromU8 } from "fflate"; +import { load as loadIon } from "ion-js"; +import { ReservoirSampler } from "../utils/ReservoirSampler.js"; +import { resolveRenderLabels } from "../utils/resolveRenderLabels.js"; +import type { ThumbnailRenderOutput } from "../utils/thumbnailRenderRep.js"; +import { THUMB_BUCKET, THUMBNAIL_BROKEN_METAS } from "../utils/thumbnailConfig.js"; +import { decompressGameState } from "../utils/gameState.js"; +import { skipCompletedGameWithoutState } from "../utils/completedGameRec.js"; +import i18next from "i18next"; +import type { i18n } from "i18next"; +import type { BasicRec, GameRec } from "types/index.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({ region: REGION }); +const DUMP_BUCKET = "abstractplay-db-dump"; +const RENDER_BUCKET = process.env.RENDER_BUCKET; +const THUMBNAIL_CACHE_CONTROL = "public, max-age=86400"; +const MIN_MOVES = 5; + +type SamplerEntry = { + active: ReservoirSampler; + completed: ReservoirSampler; +}; + +export const handler: Handler = async () => { + const i18nInstance = i18next as unknown as i18n; + await i18nInstance + .init({ + lng: "en", + fallbackLng: "en", + debug: true, + }) + .then(async () => { + if (!i18nInstance.isInitialized) { + throw new Error("i18n is not initialized where it should be!"); + } + const gamesI18n = addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + + const gameInfoProd = ([...gameinfo.values()] as APGamesInformation[]).filter( + (rec) => !rec.flags.includes("experimental"), + ); + + const command = new ListObjectsV2Command({ + Bucket: DUMP_BUCKET, + }); + + const allContents: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(command); + if (Contents === undefined) { + throw new Error("Could not list the bucket contents"); + } + allContents.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + command.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + } + + const manifests = allContents.filter((c) => c.Key?.includes("manifest-summary.json")); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + + const uid = match[1]; + const dataFiles = allContents.filter( + (c) => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz"), + ); + console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`); + + const samplerMap = new Map(); + for (const file of dataFiles) { + console.log(`Loading ${file.Key}`); + const getCmd = new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + }); + + try { + const response = await s3.send(getCmd); + const bytes = await response.Body?.transformToByteArray(); + if (bytes !== undefined) { + const ion = gunzipSync(bytes); + console.log(`Processing ${ion.length} bytes`); + let sofar = ""; + let ptr = 0; + const chunk = 1000000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes("}}\n")) { + const idx = sofar.indexOf("}}\n"); + const line = sofar.substring(0, idx + 2); + sofar = sofar.substring(idx + 3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + console.log( + `Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`, + ); + } else { + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + const rec = json.Item; + if (rec.pk === "GAME") { + const [meta, cbit] = rec.sk.split("#"); + if (cbit === "1" && skipCompletedGameWithoutState(rec)) { + continue; + } + if (rec.state === undefined || rec.state === "") { + continue; + } + const g = GameFactory(meta, decompressGameState(rec.state)); + if (g === undefined) { + throw new Error( + `Error instantiating the following game record:\n${rec}`, + ); + } + const numMoves = g.stack.length; + if (numMoves >= MIN_MOVES) { + if (samplerMap.has(meta)) { + const sampler = samplerMap.get(meta)!; + if (cbit === "1") { + sampler.completed.add(rec as GameRec); + } else { + sampler.active.add(rec as GameRec); + } + } else { + const sampler: SamplerEntry = { + completed: new ReservoirSampler(), + active: new ReservoirSampler(), + }; + if (cbit === "1") { + sampler.completed.add(rec as GameRec); + } else { + sampler.active.add(rec as GameRec); + } + samplerMap.set(meta, sampler); + } + } + } + } + } catch (err) { + console.log(`An error occurred while loading an ION record: ${line}`); + console.error(err); + } + } + ptr += chunk; + } + } else { + throw new Error(`Could not load bytes from ${file.Key}`); + } + } catch (err) { + console.log( + `An error occured while reading data files. The specific file was ${JSON.stringify(file)}`, + ); + console.error(err); + } + } + console.log("GAME records processed"); + + const allRecs = new Map(); + for (const [meta, entry] of samplerMap.entries()) { + const active = entry.active.getSample(); + let rec: GameRec; + if (active.length > 0) { + rec = active[0]; + } else { + const completed = entry.completed.getSample(); + if (completed.length === 0) { + console.log(`No active or completed games found for meta "${meta}"! Failsafe needed.`); + continue; + } + rec = completed[0]; + } + let g = GameFactory(meta, decompressGameState(rec.state)); + if (g === undefined) { + throw new Error(`Error instantiating the following game record:\n${rec}`); + } + const stripped = g.serialize({ strip: true }); + g = GameFactory(meta, stripped); + if (g === undefined) { + throw new Error( + `Error instantiating the following game record AFTER STRIPPING:\n${rec}`, + ); + } + const rep = g.render({}) as ThumbnailRenderOutput; + const resolved = resolveRenderLabels(rep, rec.players, (key, params) => + String(gamesI18n.t(key, params ?? {})), + ); + allRecs.set(meta, resolved); + } + console.log(`Generated ${allRecs.size} thumbnails`); + + const metasProd = gameInfoProd.map((rec) => rec.uid); + const keys = [...allRecs.keys()].filter((id) => !metasProd.includes(id)); + if (keys.length > 0) { + console.log( + `${keys.length} production games do not have active or completed game records, and so no thumbnail was generated: ${JSON.stringify(keys)}`, + ); + } + + for (const [meta, rep] of allRecs.entries()) { + const body = JSON.stringify(rep); + let cmd = new PutObjectCommand({ + Bucket: THUMB_BUCKET, + Key: `${meta}.json`, + Body: body, + CacheControl: THUMBNAIL_CACHE_CONTROL, + ContentType: "application/json", + }); + let response = await s3.send(cmd); + if (response["$metadata"].httpStatusCode !== 200) { + console.log(response); + } + if (!THUMBNAIL_BROKEN_METAS.includes(meta)) { + cmd = new PutObjectCommand({ + Bucket: RENDER_BUCKET, + Key: `${meta}.json`, + Body: body, + CacheControl: THUMBNAIL_CACHE_CONTROL, + ContentType: "application/json", + }); + response = await s3.send(cmd); + if (response["$metadata"].httpStatusCode !== 200) { + console.log(response); + } + } + } + console.log("Thumbnails stored"); + console.log("ALL DONE"); + }) + .catch((err) => { + throw new Error(`An error occurred (final catch):\n${err}`); + }); +}; diff --git a/crons/src/functions/tournament-data.ts b/crons/src/functions/tournament-data.ts new file mode 100644 index 00000000..6c5ad378 --- /dev/null +++ b/crons/src/functions/tournament-data.ts @@ -0,0 +1,308 @@ +'use strict'; + +import { S3Client, GetObjectCommand, ListObjectsV2Command, type _Object } from "@aws-sdk/client-s3"; +import { Handler } from "aws-lambda"; +import { gunzipSync, strFromU8 } from "fflate"; +import { load as loadIon } from "ion-js"; +import { putRecordsJson } from "../utils/recordsJson.js"; + +const REGION = "us-east-1"; +const s3 = new S3Client({region: REGION}); +const DUMP_BUCKET = "abstractplay-db-dump"; +const REC_BUCKET = "records.abstractplay.com"; + +type BasicRec = { + Item: { + pk: string; + sk: string; + [key: string]: any; + } +} + +type Division = { + numGames: number; + numCompleted: number; + processed: boolean; + winnerid?: string; + winner?: string; +}; + +type TournamentPlayer = { + pk: string; + sk: string; + playerid: string; + playername: string; + once?: boolean; + division?: number; + score?: number; + tiebreak?: number; + rating?: number; + timeout?: boolean; +}; + +type Tournament = { + pk: string; + sk: string; + id: string; + metaGame: string; + variants: string[]; + number: number; + started: boolean; + dateCreated: number; + datePreviousEnded: number; // 0 means either the first tournament or a restart of the series (after it stopped because not enough participants), 3000000000000 means previous tournament still running. + nextid?: string; + dateStarted?: number; + dateEnded?: number; + divisions?: { + [division: number]: Division; + }; + players?: TournamentPlayer[]; // only on archived tournaments + waiting?: boolean; // tournament does not yet have 4 players +}; + +type ResultsNode = { + pid: string; + tid: string; + metaGame: string; + won: boolean; + t50: boolean; + score: number; +}; + +type TournamentNode = { + pid: string; + tid: string; + metaGame: string; + variants: string[]; + dateEnded: number; + archived: boolean; + place: number; + participants: number; + score: number; +}; + +type SummaryNode = { + player: string; + count: number; + won: number; + t50: number; + scoreSum: number; + scoreAvg: number; + scoreMed: number; +}; + +type Summary = SummaryNode[]; + +export const handler: Handler = async (event: any, context?: any) => { + // scan bucket for data folder + const command = new ListObjectsV2Command({ + Bucket: DUMP_BUCKET, + }); + + const allContents: _Object[] = []; + try { + let isTruncatedOuter = true; + + while (isTruncatedOuter) { + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = + await s3.send(command); + if (Contents === undefined) { + throw new Error(`Could not list the bucket contents`); + } + allContents.push(...Contents); + isTruncatedOuter = IsTruncatedInner || false; + command.input.ContinuationToken = NextContinuationToken; + } + } catch (err) { + console.error(err); + } + + // find the latest `manifest-summary.json` file + const manifests = allContents.filter(c => c.Key?.includes("manifest-summary.json")); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + // from there, extract the UID and list of associated data files + const uid = match[1]; + const dataFiles = allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz")); + console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`); + + // load the data from each data file, but only keep the COMPLETEDTOURNAMENT records + const tourneys: Tournament[] = []; + let possPlayers: TournamentPlayer[]|undefined = []; + let possTourneys: Tournament[]|undefined = []; + const archivedTourneys = new Set(); + for (const file of dataFiles) { + console.log(`Loading ${file.Key}`); + const command = new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + }); + + try { + const response = await s3.send(command); + // The Body object also has 'transformToByteArray' and 'transformToWebStream' methods. + const bytes = await response.Body?.transformToByteArray(); + if (bytes !== undefined) { + const ion = gunzipSync(bytes); + console.log(`Processing ${ion.length} bytes`); + let sofar = ""; + let ptr = 0; + const chunk = 1000000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes("}}\n")) { + const idx = sofar.indexOf("}}\n"); + const line = sofar.substring(0, idx+2); + sofar = sofar.substring(idx+3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + console.log(`Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`) + } else { + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + const rec = json.Item; + if (rec.pk === "COMPLETEDTOURNAMENT") { + tourneys.push(rec as Tournament); + archivedTourneys.add((rec as Tournament).id); + } else if (rec.pk === "TOURNAMENT" && (rec as Tournament).dateEnded !== undefined) { + possTourneys.push(rec as Tournament); + } else if (rec.pk === "TOURNAMENTPLAYER") { + possPlayers.push(rec as TournamentPlayer); + } + } + } catch (err) { + console.log(`An error occurred while loading an ION record: ${line}`); + console.error(err); + } + } + ptr += chunk; + } + } + } catch (err) { + console.log(`An error occured while reading data files. The specific file was ${JSON.stringify(file)}`) + console.error(err); + } + } + console.log(`Found ${tourneys.length} COMPLETEDTOURNAMENT records`); + + // for each possTourney, merge matching players, and add to overall tourneys list + for (const tourney of possTourneys) { + const players = possPlayers.filter(rec => rec.sk.startsWith(tourney.id)); + const newrec = structuredClone(tourney) as Tournament; + newrec.players = players; + tourneys.push(newrec); + } + possTourneys = undefined; + possPlayers = undefined; + + // for each tournament, tabulate results + const pushToMap = (m: Map, key: string, value: any) => { + if (m.has(key)) { + const current = m.get(key)!; + m.set(key, [...current, value]); + } else { + m.set(key, [value]); + } + } + const sortPlayers = (a: TournamentPlayer, b: TournamentPlayer): number => { + if (a.score === b.score) { + if (a.tiebreak === b.tiebreak) { + return b.rating! - a.rating!; + } else { + return b.tiebreak! - a.tiebreak!; + } + } else { + return b.score! - a.score!; + } + } + + const summary = new Map(); + const individual = new Map(); + for (const tourney of tourneys) { + for (const [nstr, division] of Object.entries(tourney.divisions!)) { + const num = parseInt(nstr, 10); + const players = tourney.players!.filter(p => p.division === num); + players.sort(sortPlayers); + if (players[0].playerid !== division.winnerid) { + console.log(`Tournament winners differed for division ${nstr}:\nSorted says ${players[0].playerid}, division says ${division.winnerid}\n${JSON.stringify(tourney)}`); + } + for (let p = 1; p <= players.length; p++) { + const player = players[p-1]; + let won = false; + if (p === 1) { + won = true; + } + let t50 = false; + if (p < (players.length / 2)) { + t50 = true; + } + const score = 100 * ((players.length - p) / (players.length - 1)); + const result: ResultsNode = { + pid: player.playerid, + tid: tourney.id, + metaGame: tourney.metaGame, + won, + t50, + score, + } + const ind: TournamentNode = { + pid: player.playerid, + tid: tourney.id, + metaGame: tourney.metaGame, + variants: tourney.variants, + archived: archivedTourneys.has(tourney.id), + dateEnded: tourney.dateEnded!, + place: p, + participants: players.length, + score, + }; + pushToMap(summary, player.playerid, result); + pushToMap(individual, player.playerid, ind); + } + } + } + console.log(`tournament-data: ${summary.size} summary entries; ${individual.size} individual entries`); + + // tabulate the summaries + const finalSummary: Summary = []; + for (const [player, entries] of summary.entries()) { + const count = entries.length; + const won = entries.filter(r => r.won).length; + const t50 = entries.filter(r => r.t50).length; + const scores = entries.map(r => r.score); + const scoreSum = scores.reduce((acc, curr) => acc + curr, 0); + const scoreAvg = scoreSum / scores.length; + scores.sort((a, b) => a - b); + let scoreMed: number; + if (scores.length % 2 === 0) { + const rightIdx = scores.length / 2; + const leftIdx = rightIdx - 1; + scoreMed = (scores[leftIdx] + scores[rightIdx]) / 2; + } else { + scoreMed = scores[Math.floor(scores.length / 2)]; + } + finalSummary.push({ + player, + count, + won, + t50, + scoreSum, + scoreAvg, + scoreMed + }); + } + + await putRecordsJson(s3, "tournament-summary.json", finalSummary); + console.log("Summary data done"); + + for (const [player, lst] of individual.entries()) { + await putRecordsJson(s3, `player/tournaments/${player}.json`, lst); + } + console.log("Individual data done"); + + console.log("ALL DONE"); +}; diff --git a/crons/src/lib/activeGamesForUser.test.ts b/crons/src/lib/activeGamesForUser.test.ts new file mode 100644 index 00000000..dde52307 --- /dev/null +++ b/crons/src/lib/activeGamesForUser.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; +import { isActiveDashboardGame } from './activeGamesForUser.js'; + +describe('isActiveDashboardGame', () => { + it('is active when toMove is set', () => { + expect(isActiveDashboardGame({ toMove: '0' })).toBe(true); + expect(isActiveDashboardGame({ toMove: [true, false] })).toBe(true); + }); + + it('is not active when toMove is empty or missing', () => { + expect(isActiveDashboardGame({ toMove: '' })).toBe(false); + expect(isActiveDashboardGame({})).toBe(false); + expect(isActiveDashboardGame({ toMove: null })).toBe(false); + }); +}); diff --git a/crons/src/lib/activeGamesForUser.ts b/crons/src/lib/activeGamesForUser.ts new file mode 100644 index 00000000..17c64887 --- /dev/null +++ b/crons/src/lib/activeGamesForUser.ts @@ -0,0 +1,45 @@ +import { + DynamoDBDocumentClient, + QueryCommand, +} from '@aws-sdk/lib-dynamodb'; + +export type CurrentGameRow = { + sk: string; + id?: string; + metaGame: string; + variants?: string[]; + toMove?: string | boolean[] | null; +}; + +export function isActiveDashboardGame(game: { toMove?: string | boolean[] | null }): boolean { + return game.toMove !== '' && game.toMove !== null && game.toMove !== undefined; +} + +export async function listActiveCurrentGames( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, +): Promise { + const items: CurrentGameRow[] = []; + let lastEvaluatedKey: Record | undefined; + + do { + const page = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + ExpressionAttributeValues: { ':pk': `CURRENTGAMES#${userId}` }, + ExclusiveStartKey: lastEvaluatedKey, + })); + + for (const item of page.Items ?? []) { + const row = item as CurrentGameRow; + if (isActiveDashboardGame(row)) { + items.push(row); + } + } + lastEvaluatedKey = page.LastEvaluatedKey; + } while (lastEvaluatedKey); + + return items; +} diff --git a/crons/src/lib/apbackI18n.ts b/crons/src/lib/apbackI18n.ts new file mode 100644 index 00000000..c72afcb1 --- /dev/null +++ b/crons/src/lib/apbackI18n.ts @@ -0,0 +1,58 @@ +import i18n from 'i18next'; +import { applyGameslibBundlesTo, GAMESLIB_APGAMES_LANGS } from './gameslibLocales.js'; +import en from '../locales/en/apback.json'; +import fr from '../locales/fr/apback.json'; +import de from '../locales/de/apback.json'; +import it from '../locales/it/apback.json'; +import esUS from '../locales/es-US/apback.json'; +import pt from '../locales/pt/apback.json'; +import ta from '../locales/ta/apback.json'; + +const LOCALE_RESOURCES = { en, fr, de, it, 'es-US': esUS, pt, ta } as const; +const REGISTERED_LANGUAGES = [ + ...new Set([...Object.keys(LOCALE_RESOURCES), ...GAMESLIB_APGAMES_LANGS]), +]; + +export function resolvePlayerLanguage(language: string | undefined): string { + if (language && REGISTERED_LANGUAGES.includes(language)) { + return language; + } + if (language) { + const lower = language.toLowerCase(); + if (lower === 'es' || lower.startsWith('es-')) { + return 'es-US'; + } + } + return 'en'; +} + +export async function changeLanguageForPlayer(player: { + language: string | undefined; +}): Promise { + const lng = resolvePlayerLanguage(player.language); + if (i18n.language !== lng) { + await i18n.changeLanguage(lng); + } +} + +/** Init i18next with vendored apback + gameslib locale bundles (email/push copy). */ +export async function initApbackI18n(language = 'en'): Promise { + await i18n.init({ + lng: language, + fallbackLng: 'en', + resources: Object.fromEntries( + REGISTERED_LANGUAGES.map((lng) => [ + lng, + { + ...(lng in LOCALE_RESOURCES + ? { translation: LOCALE_RESOURCES[lng as keyof typeof LOCALE_RESOURCES] } + : {}), + }, + ]), + ), + }); + applyGameslibBundlesTo(i18n); +} + +/** @deprecated Use initApbackI18n */ +export const initi18n = initApbackI18n; diff --git a/crons/src/lib/batchRatings.test.ts b/crons/src/lib/batchRatings.test.ts new file mode 100644 index 00000000..718161ec --- /dev/null +++ b/crons/src/lib/batchRatings.test.ts @@ -0,0 +1,164 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; +import { + assignTournamentPlayerRatings, + batchRatingGameLabel, + buildPlayerCountsByUid, + compareBatchRatings, + defaultGlickoPrior, + GLICKO_PRIOR_RATING_LOW, + lookupBatchRating, + parseBatchRatingGameLabel, +} from "./batchRatings.js"; +import { GLICKO_RATING_START, GLICKO_RD_START } from "../functions/summarizeHelpers.js"; + +const fixtureDir = join(dirname(fileURLToPath(import.meta.url)), "../../test/fixtures"); +const fixture = JSON.parse(readFileSync(join(fixtureDir, "batch-ratings.json"), "utf8")) as { + highest: UserGameRating[]; +}; + +describe("batchRatingGameLabel", () => { + it("labels no variants", () => { + expect(batchRatingGameLabel("chess", [])).toBe("chess (no variants)"); + }); + + it("labels sorted variant UIDs", () => { + expect(batchRatingGameLabel("go", ["handicap", "9x9"])).toBe("go (9x9|handicap)"); + }); +}); + +describe("parseBatchRatingGameLabel", () => { + it("parses no variants", () => { + expect(parseBatchRatingGameLabel("chess (no variants)")).toEqual({ + metaUid: "chess", + variantUids: [], + }); + }); + + it("parses variant UIDs", () => { + expect(parseBatchRatingGameLabel("go (9x9|handicap)")).toEqual({ + metaUid: "go", + variantUids: ["9x9", "handicap"], + }); + }); + + it("round-trips batchRatingGameLabel", () => { + const label = batchRatingGameLabel("go", ["handicap", "9x9"]); + expect(parseBatchRatingGameLabel(label)).toEqual({ + metaUid: "go", + variantUids: ["9x9", "handicap"], + }); + }); +}); + +describe("defaultGlickoPrior", () => { + it("uses 1200/350 start aligned with batch Elo", () => { + const prior = defaultGlickoPrior(); + expect(prior.rating).toBe(GLICKO_RATING_START); + expect(prior.rd).toBe(GLICKO_RD_START); + expect(prior.ratingLow).toBe(GLICKO_RATING_START - 2 * GLICKO_RD_START); + }); +}); + +describe("lookupBatchRating", () => { + it("finds an existing row", () => { + const row = lookupBatchRating(fixture.highest, "chess", [], "alice"); + expect(row.user).toBe("alice"); + expect(row.glicko?.ratingLow).toBe(1200); + }); + + it("returns prior for missing user", () => { + const row = lookupBatchRating(fixture.highest, "chess", [], "unknown"); + expect(row.glicko?.rating).toBe(1200); + expect(row.glicko?.ratingLow).toBe(500); + }); + + it("matches variant label keys", () => { + const row = lookupBatchRating(fixture.highest, "go", ["9x9", "handicap"], "carol"); + expect(row.glicko?.ratingLow).toBe(1170); + }); +}); + +describe("compareBatchRatings", () => { + it("sorts by ratingLow descending", () => { + const sorted = [...fixture.highest.filter((r) => r.game === "chess (no variants)")].sort( + compareBatchRatings, + ); + expect(sorted[0]!.user).toBe("alice"); + expect(sorted[1]!.user).toBe("bob"); + }); + + it("tie-breaks lower rd before higher raw rating", () => { + const a: UserGameRating = { + user: "a", + game: "Test (no variants)", + rating: 1200, + wld: [0, 0, 0], + glicko: { + rating: 1300, + rd: 50, + volatility: 0.06, + ratingLow: 1200, + ratingHigh: 1400, + provisional: false, + established: false, + n: 5, + }, + }; + const b: UserGameRating = { + user: "b", + game: "Test (no variants)", + rating: 1200, + wld: [0, 0, 0], + glicko: { + rating: 1350, + rd: 75, + volatility: 0.06, + ratingLow: 1200, + ratingHigh: 1500, + provisional: false, + established: false, + n: 5, + }, + }; + expect(compareBatchRatings(a, b)).toBeLessThan(0); + }); +}); + +describe("buildPlayerCountsByUid", () => { + it("counts distinct users per meta uid", () => { + const counts = buildPlayerCountsByUid(fixture.highest); + expect(counts).toEqual({ chess: 2, go: 2 }); + }); +}); + +describe("assignTournamentPlayerRatings", () => { + it("orders players by glicko ratingLow descending", () => { + const players = [ + { playerid: "bob", playername: "Bob" }, + { playerid: "alice", playername: "Alice" }, + { playerid: "unknown", playername: "Unknown" }, + ]; + assignTournamentPlayerRatings(players, fixture.highest, "chess", []); + players.sort((a, b) => b.rating! - a.rating!); + expect(players.map((p) => p.playerid)).toEqual(["alice", "bob", "unknown"]); + expect(players[0]!.rating).toBe(1200); + expect(players[1]!.rating).toBe(1090); + expect(players[2]!.rating).toBe(GLICKO_PRIOR_RATING_LOW); + }); + + it("uses variant-aware game labels", () => { + const players = [ + { playerid: "carol" }, + { playerid: "alice" }, + ]; + assignTournamentPlayerRatings(players, fixture.highest, "go", ["handicap", "9x9"]); + players.sort((a, b) => b.rating! - a.rating!); + expect(players[0]!.playerid).toBe("alice"); + expect(players[0]!.rating).toBe(1200); + expect(players[1]!.rating).toBe(1170); + }); +}); diff --git a/crons/src/lib/batchRatings.ts b/crons/src/lib/batchRatings.ts new file mode 100644 index 00000000..da3ef1aa --- /dev/null +++ b/crons/src/lib/batchRatings.ts @@ -0,0 +1,133 @@ +import { gameinfo, variantUidsForBatchRating } from "@abstractplay/gameslib"; +import type { GlickoStats } from "types/stats/GlickoStats.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; +import { + GLICKO_RATING_START, + GLICKO_RD_START, + GLICKO_VOLATILITY_START, + toGlickoStats, +} from "../functions/summarizeHelpers.js"; + +export const GLICKO_PRIOR_RATING_LOW = GLICKO_RATING_START - 2 * GLICKO_RD_START; + +export function glickoConservativeSortKey(row: UserGameRating): number { + return row.glicko?.ratingLow ?? GLICKO_PRIOR_RATING_LOW; +} + +export type TournamentSeedPlayer = { + playerid: string; + rating?: number; + score?: number; +}; + +/** Assign batch Glicko conservative sort keys for tournament division seeding. */ +export function assignTournamentPlayerRatings( + players: TournamentSeedPlayer[], + highest: UserGameRating[], + metaUid: string, + variants: string[], +): void { + for (const player of players) { + const row = lookupBatchRating(highest, metaUid, variants, player.playerid); + player.rating = glickoConservativeSortKey(row); + player.score = 0; + } +} + +/** Meta UID + variant UIDs → summarize `highest[].game` key. */ +export function batchRatingGameLabel(metaUid: string, variants: string[]): string { + if (variants.length === 0) { + return `${metaUid} (no variants)`; + } + const sorted = [...variants].sort(); + return `${metaUid} (${sorted.join("|")})`; +} + +export function defaultGlickoPrior(): GlickoStats { + return toGlickoStats(GLICKO_RATING_START, GLICKO_RD_START, GLICKO_VOLATILITY_START, 0); +} + +export function lookupBatchRating( + highest: UserGameRating[], + metaUid: string, + variants: string[], + userId: string, + playerCount = 2, +): UserGameRating { + const defs = gameinfo.get(metaUid)?.variants; + const canonical = + defs !== undefined && defs.length > 0 + ? variantUidsForBatchRating(metaUid, playerCount, variants) + : variants; + const game = batchRatingGameLabel(metaUid, canonical); + const row = highest.find((r) => r.user === userId && r.game === game); + if (row !== undefined) { + return row; + } + return { + user: userId, + game, + rating: GLICKO_RATING_START, + wld: [0, 0, 0], + glicko: defaultGlickoPrior(), + }; +} + +/** Sort key: `ratingLow` desc → lower `rd` → higher raw `glicko.rating`. */ +export function compareBatchRatings(a: UserGameRating, b: UserGameRating): number { + const priorLow = GLICKO_RATING_START - 2 * GLICKO_RD_START; + const lowA = a.glicko?.ratingLow ?? priorLow; + const lowB = b.glicko?.ratingLow ?? priorLow; + if (lowB !== lowA) { + return lowB - lowA; + } + const rdA = a.glicko?.rd ?? GLICKO_RD_START; + const rdB = b.glicko?.rd ?? GLICKO_RD_START; + if (rdA !== rdB) { + return rdA - rdB; + } + const ratingA = a.glicko?.rating ?? GLICKO_RATING_START; + const ratingB = b.glicko?.rating ?? GLICKO_RATING_START; + return ratingB - ratingA; +} + +/** Meta UID prefix from a `highest[].game` label. */ +export function metaUidFromRatingGameLabel(game: string): string { + const paren = game.indexOf(" ("); + return paren === -1 ? game : game.slice(0, paren); +} + +const NO_VARIANTS_SUFFIX = "no variants"; + +/** Parse `batchRatingGameLabel` output into meta UID + variant UIDs. */ +export function parseBatchRatingGameLabel(gameLabel: string): { metaUid: string; variantUids: string[] } { + const metaUid = metaUidFromRatingGameLabel(gameLabel); + const paren = gameLabel.indexOf(" ("); + if (paren === -1) { + return { metaUid, variantUids: [] }; + } + const inner = gameLabel.endsWith(")") ? gameLabel.slice(paren + 2, -1) : ""; + if (inner === NO_VARIANTS_SUFFIX) { + return { metaUid, variantUids: [] }; + } + return { metaUid, variantUids: inner ? inner.split("|") : [] }; +} + +/** Distinct rated users per meta UID from `highest[]` game labels. */ +export function buildPlayerCountsByUid(highest: UserGameRating[]): Record { + const usersByUid = new Map>(); + for (const row of highest) { + const uid = metaUidFromRatingGameLabel(row.game); + let users = usersByUid.get(uid); + if (users === undefined) { + users = new Set(); + usersByUid.set(uid, users); + } + users.add(row.user); + } + const counts: Record = {}; + for (const [uid, users] of usersByUid) { + counts[uid] = users.size; + } + return counts; +} diff --git a/crons/src/lib/challengeRevokedNotifications.ts b/crons/src/lib/challengeRevokedNotifications.ts new file mode 100644 index 00000000..0531b8c4 --- /dev/null +++ b/crons/src/lib/challengeRevokedNotifications.ts @@ -0,0 +1,201 @@ +import { GetCommand, PutCommand, type DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { SendEmailCommand, SESClient } from '@aws-sdk/client-ses'; +import i18n from 'i18next'; +import { localizedGameName } from './gameDisplayName.js'; +import { isBotId, isValidUserId } from './inactiveChallengeDiscovery.js'; +import { + wantsInAppNotification, + type InAppNotificationUserSettings, +} from './inAppNotificationPrefs.js'; +import { sendPush } from './pushSubscriptions.js'; +import type { ChallengePlayer, RevokeChallengeRecord } from './revokeChallenge.js'; + +const NOTIFICATION_PK_PREFIX = 'NOTIFICATION#'; +const NOTIFICATION_INITIAL_TTL_DAYS = 180; +const SEC_PER_DAY = 86_400; + +type NotificationUser = { + id: string; + name: string; + email?: string; + language?: string; + settings?: InAppNotificationUserSettings; +}; + +function notificationPk(userId: string): string { + return `${NOTIFICATION_PK_PREFIX}${userId}`; +} + +function notificationInitialExpiresAt(now = Date.now()): number { + return Math.floor(now / 1000) + NOTIFICATION_INITIAL_TTL_DAYS * SEC_PER_DAY; +} + +function uniqueSortKey(now = Date.now()): string { + return `${now}#${Math.random().toString(36).slice(2, 10)}`; +} + +function wantsChallengeEmail(settings: InAppNotificationUserSettings | undefined): boolean { + const notifications = settings?.all?.notifications as { challenges?: boolean } | undefined; + return notifications === undefined || notifications.challenges !== false; +} + +export function createSendEmailCommand( + toAddress: string, + player: string, + subject: string, + body: string, +) { + const fullbody = `${i18n.t('DearPlayer', { player })}\r\n\r\n${body}\r\n\r\n${i18n.t('EmailOut')}`; + return new SendEmailCommand({ + Destination: { ToAddresses: [toAddress] }, + Message: { + Body: { Text: { Charset: 'UTF-8', Data: fullbody } }, + Subject: { Charset: 'UTF-8', Data: subject }, + }, + Source: 'abstractplay@mail.abstractplay.com', + }); +} + +async function loadNotificationUser( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: userId }, + ProjectionExpression: 'id, #name, email, #language, settings', + ExpressionAttributeNames: { '#name': 'name', '#language': 'language' }, + })); + if (data.Item === undefined) { + return undefined; + } + const item = data.Item; + return { + id: String(item.id ?? userId), + name: String(item.name ?? userId), + email: typeof item.email === 'string' ? item.email : undefined, + language: typeof item.language === 'string' ? item.language : undefined, + settings: item.settings as InAppNotificationUserSettings | undefined, + }; +} + +async function changeLanguageForPlayer(player: NotificationUser): Promise { + const lng = player.language ?? 'en'; + if (i18n.language !== lng) { + await i18n.changeLanguage(lng); + } +} + +async function createChallengeRevokedNotification( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, + body: { + type: 'challengeRevoked'; + challengeId: string; + metaGame: string; + revokerId: string; + revokerName: string; + }, + userSettings?: InAppNotificationUserSettings, +): Promise { + if (!wantsInAppNotification(userSettings, 'challenges')) { + return; + } + const now = Date.now(); + await client.send(new PutCommand({ + TableName: tableName, + Item: { + pk: notificationPk(userId), + sk: uniqueSortKey(now), + body, + expiresAt: notificationInitialExpiresAt(now), + }, + })); +} + +export async function notifyChallengeRevokedAcceptors( + client: DynamoDBDocumentClient, + tableName: string, + sesClient: SESClient, + challenge: RevokeChallengeRecord, + standing: boolean, +): Promise { + const acceptors = (challenge.players ?? []).filter( + p => isValidUserId(p.id) && p.id !== challenge.challenger.id, + ); + if (acceptors.length === 0) { + return; + } + + const revokerName = challenge.challenger.name ?? challenge.challenger.id; + + for (const acceptor of acceptors) { + if (await isBotId(client, tableName, acceptor.id)) { + continue; + } + const player = await loadNotificationUser(client, tableName, acceptor.id); + if (player === undefined) { + continue; + } + await changeLanguageForPlayer(player); + const localizedBody = i18n.t('ChallengeRevokedBody', { + name: revokerName, + metaGame: localizedGameName(challenge.metaGame), + }); + + if (player.email !== undefined && player.email !== '' && wantsChallengeEmail(player.settings)) { + await sesClient.send(createSendEmailCommand( + player.email, + player.name, + i18n.t('ChallengeRevokedSubject'), + localizedBody, + )); + } + + await sendPush(client, tableName, { + userId: player.id, + topic: 'challenges', + title: i18n.t('PUSH.titles.revoked'), + body: localizedBody, + url: '/', + }); + + if (!standing) { + await createChallengeRevokedNotification(client, tableName, player.id, { + type: 'challengeRevoked', + challengeId: challenge.id, + metaGame: challenge.metaGame, + revokerId: challenge.challenger.id, + revokerName, + }, player.settings); + } + } +} + +export function toRevokeChallengeRecord( + challenge: Record, +): RevokeChallengeRecord | undefined { + const id = typeof challenge.id === 'string' ? challenge.id : undefined; + const metaGame = typeof challenge.metaGame === 'string' ? challenge.metaGame : undefined; + const numPlayers = typeof challenge.numPlayers === 'number' ? challenge.numPlayers : undefined; + const challenger = challenge.challenger as ChallengePlayer | undefined; + if ( + id === undefined + || metaGame === undefined + || numPlayers === undefined + || challenger === undefined + || typeof challenger.id !== 'string' + ) { + return undefined; + } + return { + id, + metaGame, + numPlayers, + challenger, + challengees: challenge.challengees as ChallengePlayer[] | undefined, + players: challenge.players as ChallengePlayer[] | undefined, + }; +} diff --git a/crons/src/lib/gameDisplayName.test.ts b/crons/src/lib/gameDisplayName.test.ts new file mode 100644 index 00000000..f570129b --- /dev/null +++ b/crons/src/lib/gameDisplayName.test.ts @@ -0,0 +1,33 @@ +import { beforeEach, describe, expect, it } from 'vitest'; +import i18n from 'i18next'; +import { gameinfo } from '@abstractplay/gameslib'; +import { localizedGameName } from './gameDisplayName.js'; +import { changeLanguageForPlayer, initApbackI18n } from './apbackI18n.js'; + +describe('tournament email i18n', () => { + beforeEach(async () => { + if (i18n.isInitialized) { + await i18n.changeLanguage('en'); + } + await initApbackI18n('en'); + }); + + it('loads apgames bundles for managed languages', () => { + expect(i18n.hasResourceBundle('de', 'apgames')).toBe(true); + expect(i18n.hasResourceBundle('en', 'apgames')).toBe(true); + }); + + it('localizedGameName resolves after changeLanguageForPlayer', async () => { + const uid = 'hex'; + const fallback = gameinfo.get(uid)?.name ?? uid; + await changeLanguageForPlayer({ language: 'en' }); + expect(localizedGameName(uid)).toBe(fallback); + await changeLanguageForPlayer({ language: 'de' }); + expect(localizedGameName(uid).length).toBeGreaterThan(0); + }); + + it('maps es player language to es-US', async () => { + await changeLanguageForPlayer({ language: 'es' }); + expect(i18n.language).toBe('es-US'); + }); +}); diff --git a/crons/src/lib/gameDisplayName.ts b/crons/src/lib/gameDisplayName.ts new file mode 100644 index 00000000..c2ebfe66 --- /dev/null +++ b/crons/src/lib/gameDisplayName.ts @@ -0,0 +1,11 @@ +import i18n from 'i18next'; +import { gameinfo } from '@abstractplay/gameslib'; + +/** Localized meta-game title for the active i18next language (email copy). */ +export function localizedGameName(metaUid: string): string { + const key = `names.${metaUid}`; + if (i18n.exists(`apgames:${key}`)) { + return i18n.t(`apgames:${key}`); + } + return gameinfo.get(metaUid)?.name ?? metaUid; +} diff --git a/crons/src/lib/gameStartNotifications.ts b/crons/src/lib/gameStartNotifications.ts new file mode 100644 index 00000000..63fb2290 --- /dev/null +++ b/crons/src/lib/gameStartNotifications.ts @@ -0,0 +1,128 @@ +import { + DynamoDBDocumentClient, + GetCommand, + PutCommand, +} from '@aws-sdk/lib-dynamodb'; + +const NOTIFICATION_PK_PREFIX = 'NOTIFICATION#'; +const NOTIFICATION_INITIAL_TTL_DAYS = 180; +const SEC_PER_DAY = 86_400; + +export type NotificationGamePlayer = { + id: string; + name: string; +}; + +export type NotificationGame = { + id: string; + metaGame: string; + variants?: string[]; + players: NotificationGamePlayer[]; +}; + +type GameStartNotificationBody = { + type: 'gameStart'; + gameId: string; + metaGame: string; + variants: string[]; + opponentId: string; + opponentName: string; +}; + +function notificationPk(userId: string): string { + return `${NOTIFICATION_PK_PREFIX}${userId}`; +} + +function notificationInitialExpiresAt(now = Date.now()): number { + return Math.floor(now / 1000) + NOTIFICATION_INITIAL_TTL_DAYS * SEC_PER_DAY; +} + +function uniqueSortKey(now = Date.now()): string { + return `${now}#${Math.random().toString(36).slice(2, 10)}`; +} + +function gameVariants(game: NotificationGame): string[] { + return game.variants ?? []; +} + +async function isBotId( + client: DynamoDBDocumentClient, + tableName: string, + id: string, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'BOT', sk: id }, + ProjectionExpression: '#pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + })); + return data.Item !== undefined; +} + +async function filterHumanIds( + client: DynamoDBDocumentClient, + tableName: string, + ids: string[], +): Promise { + const human: string[] = []; + for (const id of ids) { + if (!(await isBotId(client, tableName, id))) { + human.push(id); + } + } + return human; +} + +function opponentForPlayer( + game: NotificationGame, + playerId: string, + humanPlayers: NotificationGamePlayer[], +): NotificationGamePlayer | undefined { + return humanPlayers.find(p => p.id !== playerId); +} + +async function createGameStartNotification( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, + body: GameStartNotificationBody, +): Promise { + if (await isBotId(client, tableName, userId)) { + return; + } + const now = Date.now(); + await client.send(new PutCommand({ + TableName: tableName, + Item: { + pk: notificationPk(userId), + sk: uniqueSortKey(now), + body, + expiresAt: notificationInitialExpiresAt(now), + }, + })); +} + +export async function enqueueGameStartNotifications( + client: DynamoDBDocumentClient, + tableName: string, + game: NotificationGame, +): Promise { + const humanIds = await filterHumanIds(client, tableName, game.players.map(p => p.id)); + const humanPlayers = game.players.filter(p => humanIds.includes(p.id)); + const variants = gameVariants(game); + + await Promise.all(humanPlayers.map(async (player) => { + const opponent = opponentForPlayer(game, player.id, humanPlayers); + if (opponent === undefined) { + return; + } + await createGameStartNotification(client, tableName, player.id, { + type: 'gameStart', + gameId: game.id, + metaGame: game.metaGame, + variants, + opponentId: opponent.id, + opponentName: opponent.name, + }); + })); +} diff --git a/crons/src/lib/gameslibLocales.ts b/crons/src/lib/gameslibLocales.ts new file mode 100644 index 00000000..c2f0e12b --- /dev/null +++ b/crons/src/lib/gameslibLocales.ts @@ -0,0 +1,36 @@ +import { createRequire } from 'module'; +import { existsSync, readFileSync } from 'fs'; +import path from 'path'; +import type { i18n } from 'i18next'; + +const require = createRequire(import.meta.url); +const gameslibRoot = path.dirname( + require.resolve('@abstractplay/gameslib/package.json'), +); +const localesPath = path.join(gameslibRoot, 'locales'); + +const GAMESLIB_NAMESPACES = ['apgames', 'apresults'] as const; + +export const GAMESLIB_APGAMES_LANGS = ['en', 'fr', 'de', 'it', 'es-US'] as const; + +/** Load gameslib locale JSON from disk (Node 24-safe; no static JSON imports). */ +export function loadGameslibLocaleBundles(lang: string): Record { + const bundles: Record = {}; + for (const ns of GAMESLIB_NAMESPACES) { + const filePath = path.join(localesPath, lang, `${ns}.json`); + if (existsSync(filePath)) { + bundles[ns] = JSON.parse(readFileSync(filePath, 'utf8')); + } + } + return bundles; +} + +/** Register gameslib apgames/apresults bundles on the host i18next instance. */ +export function applyGameslibBundlesTo(i18nInstance: i18n): void { + for (const lng of GAMESLIB_APGAMES_LANGS) { + const bundles = loadGameslibLocaleBundles(lng); + for (const [ns, data] of Object.entries(bundles)) { + i18nInstance.addResourceBundle(lng, ns, data, true, true); + } + } +} diff --git a/crons/src/lib/inAppNotificationPrefs.test.ts b/crons/src/lib/inAppNotificationPrefs.test.ts new file mode 100644 index 00000000..cc604b39 --- /dev/null +++ b/crons/src/lib/inAppNotificationPrefs.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { wantsInAppNotification } from "./inAppNotificationPrefs.js"; + +describe("wantsInAppNotification", () => { + it("defaults to true when prefs are missing", () => { + expect(wantsInAppNotification(undefined, "ratingChange")).toBe(true); + expect(wantsInAppNotification({ all: {} }, "ratingChange")).toBe(true); + }); + + it("returns false when ratingChange is disabled", () => { + expect(wantsInAppNotification({ + all: { inAppNotifications: { ratingChange: false } }, + }, "ratingChange")).toBe(false); + }); + + it("returns false when tournamentStart is disabled", () => { + expect(wantsInAppNotification({ + all: { inAppNotifications: { tournamentStart: false } }, + }, "tournamentStart")).toBe(false); + }); +}); diff --git a/crons/src/lib/inAppNotificationPrefs.ts b/crons/src/lib/inAppNotificationPrefs.ts new file mode 100644 index 00000000..5092584f --- /dev/null +++ b/crons/src/lib/inAppNotificationPrefs.ts @@ -0,0 +1,31 @@ +export type InAppNotificationCategory = + | "challenges" + | "gameStart" + | "gameEnd" + | "ratingChange" + | "eventInvitation" + | "completedGameChat" + | "tournamentStart" + | "tournamentEnd"; + +export type InAppNotificationUserSettings = { + all?: { + inAppNotifications?: Partial>; + [k: string]: unknown; + }; + [k: string]: unknown; +}; + +export function wantsInAppNotification( + settings: InAppNotificationUserSettings | undefined, + category: InAppNotificationCategory, +): boolean { + const prefs = settings?.all?.inAppNotifications; + if (prefs === undefined) { + return true; + } + if (!Object.prototype.hasOwnProperty.call(prefs, category)) { + return true; + } + return prefs[category] === true; +} diff --git a/crons/src/lib/inactiveChallengeDiscovery.test.ts b/crons/src/lib/inactiveChallengeDiscovery.test.ts new file mode 100644 index 00000000..0bcb6e05 --- /dev/null +++ b/crons/src/lib/inactiveChallengeDiscovery.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + filterRevokeCandidates, + isInactiveLastSeen, + isValidUserId, +} from './inactiveChallengeDiscovery.js'; + +describe('isValidUserId', () => { + it('rejects empty ids', () => { + expect(isValidUserId('')).toBe(false); + expect(isValidUserId('u1')).toBe(true); + }); +}); + +describe('isInactiveLastSeen', () => { + it('treats missing lastSeen as active', () => { + expect(isInactiveLastSeen(undefined, 100)).toBe(false); + expect(isInactiveLastSeen('bad', 100)).toBe(false); + }); + + it('flags old lastSeen as inactive', () => { + expect(isInactiveLastSeen(50, 100)).toBe(true); + expect(isInactiveLastSeen(150, 100)).toBe(false); + }); +}); + +describe('filterRevokeCandidates', () => { + const inactiveSet = new Set(['u-inactive']); + + it('returns standing and direct challenges from inactive issuers only', () => { + const candidates = filterRevokeCandidates( + [{ + id: 's1', + metaGame: 'chess', + challenger: { id: 'u-inactive' }, + }], + [{ + id: 'd1', + metaGame: 'go', + challenger: { id: 'u-active' }, + }, { + id: 'd2', + metaGame: 'go', + challenger: { id: 'u-inactive' }, + }], + inactiveSet, + ); + expect(candidates).toHaveLength(2); + expect(candidates.map(c => c.id).sort()).toEqual(['d2', 's1']); + }); +}); diff --git a/crons/src/lib/inactiveChallengeDiscovery.ts b/crons/src/lib/inactiveChallengeDiscovery.ts new file mode 100644 index 00000000..cd5e190e --- /dev/null +++ b/crons/src/lib/inactiveChallengeDiscovery.ts @@ -0,0 +1,155 @@ +import { GetCommand, QueryCommand, type DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { + queryAllDirectChallenges, + queryAllStandingChallenges, + type OpenChallengeRecord, +} from './standingChallengeQueries.js'; + +export type RevokeCandidate = { + kind: 'standing' | 'direct'; + metaGame: string; + id: string; + issuerId: string; + challenge: OpenChallengeRecord; +}; + +export type DiscoveryResult = { + inactiveUsers: number; + openChallengesScanned: number; + candidates: RevokeCandidate[]; +}; + +export function isInactiveLastSeen(lastSeen: unknown, inactiveBeforeMs: number): boolean { + return typeof lastSeen === 'number' && lastSeen < inactiveBeforeMs; +} + +export async function buildInactiveIssuerSet( + client: DynamoDBDocumentClient, + tableName: string, + inactiveBeforeMs: number, +): Promise> { + const inactive = new Set(); + let lastKey: Record | undefined; + do { + const result = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + ExpressionAttributeValues: { ':pk': 'USERS' }, + ProjectionExpression: 'sk, lastSeen', + ExclusiveStartKey: lastKey, + })); + for (const item of result.Items ?? []) { + const userId = item.sk; + if (typeof userId === 'string' && isInactiveLastSeen(item.lastSeen, inactiveBeforeMs)) { + inactive.add(userId); + } + } + lastKey = result.LastEvaluatedKey; + } while (lastKey !== undefined); + return inactive; +} + +function challengeId(item: OpenChallengeRecord): string | undefined { + if (typeof item.id === 'string') { + return item.id; + } + if (typeof item.sk === 'string') { + return item.sk; + } + return undefined; +} + +function toCandidate( + kind: 'standing' | 'direct', + item: OpenChallengeRecord, + inactiveSet: Set, +): RevokeCandidate | undefined { + const issuerId = item.challenger?.id; + const id = challengeId(item); + const metaGame = item.metaGame; + if (typeof issuerId !== 'string' || typeof id !== 'string' || typeof metaGame !== 'string') { + return undefined; + } + if (!inactiveSet.has(issuerId)) { + return undefined; + } + return { kind, metaGame, id, issuerId, challenge: item }; +} + +export function filterRevokeCandidates( + standingChallenges: OpenChallengeRecord[], + directChallenges: OpenChallengeRecord[], + inactiveSet: Set, +): RevokeCandidate[] { + const candidates: RevokeCandidate[] = []; + for (const item of standingChallenges) { + const candidate = toCandidate('standing', item, inactiveSet); + if (candidate !== undefined) { + candidates.push(candidate); + } + } + for (const item of directChallenges) { + const candidate = toCandidate('direct', item, inactiveSet); + if (candidate !== undefined) { + candidates.push(candidate); + } + } + return candidates; +} + +export async function discoverInactiveIssuerChallenges( + client: DynamoDBDocumentClient, + tableName: string, + inactiveBeforeMs: number, +): Promise { + const inactiveSet = await buildInactiveIssuerSet(client, tableName, inactiveBeforeMs); + const [standingChallenges, directChallenges] = await Promise.all([ + queryAllStandingChallenges(client, tableName), + queryAllDirectChallenges(client, tableName), + ]); + const candidates = filterRevokeCandidates(standingChallenges, directChallenges, inactiveSet); + return { + inactiveUsers: inactiveSet.size, + openChallengesScanned: standingChallenges.length + directChallenges.length, + candidates, + }; +} + +export async function issuerStillInactive( + client: DynamoDBDocumentClient, + tableName: string, + issuerId: string, + inactiveBeforeMs: number, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'USERS', sk: issuerId }, + ProjectionExpression: 'lastSeen', + })); + if (data.Item === undefined) { + return false; + } + return isInactiveLastSeen(data.Item.lastSeen, inactiveBeforeMs); +} + +export function isValidUserId(userId: string): boolean { + return userId.length > 0; +} + +export async function isBotId( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, +): Promise { + if (!isValidUserId(userId)) { + return false; + } + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'BOT', sk: userId }, + ProjectionExpression: '#pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + })); + return data.Item !== undefined; +} diff --git a/crons/src/lib/pauseRealStanding.test.ts b/crons/src/lib/pauseRealStanding.test.ts new file mode 100644 index 00000000..a91f9210 --- /dev/null +++ b/crons/src/lib/pauseRealStanding.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; +import { GetCommand, PutCommand, type DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; +import { pauseMatchingRealStandingEntries } from './pauseRealStanding.js'; + +describe('pauseMatchingRealStandingEntries', () => { + it('suspends only matching preset entries', async () => { + let putItem: Record | undefined; + const client = { + async send(command: GetCommand | PutCommand) { + if (command instanceof GetCommand) { + return { + Item: { + pk: 'REALSTANDING', + sk: 'u1', + standing: [{ + id: 'p1', + metaGame: 'chess', + numPlayers: 2, + variants: ['standard'], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + limit: 1, + sensitivity: 'variants', + suspended: false, + }, { + id: 'p2', + metaGame: 'go', + numPlayers: 2, + variants: [], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + limit: 1, + sensitivity: 'meta', + suspended: false, + }], + }, + }; + } + putItem = command.input.Item as Record; + return {}; + }, + } as DynamoDBDocumentClient; + + const paused = await pauseMatchingRealStandingEntries(client, 'table', 'u1', { + metaGame: 'chess', + numPlayers: 2, + variants: ['standard'], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + }); + + expect(paused).toBe(1); + const standing = (putItem?.standing as { id: string; suspended: boolean }[]); + expect(standing[0]?.suspended).toBe(true); + expect(standing[1]?.suspended).toBe(false); + }); +}); diff --git a/crons/src/lib/pauseRealStanding.ts b/crons/src/lib/pauseRealStanding.ts new file mode 100644 index 00000000..a9a25af2 --- /dev/null +++ b/crons/src/lib/pauseRealStanding.ts @@ -0,0 +1,50 @@ +import { DynamoDBDocumentClient, GetCommand, PutCommand } from '@aws-sdk/lib-dynamodb'; +import { + challengeMatchesStandingEntry, + type ChallengeForStandingMatch, + type StandingPresetEntry, +} from './standingChallengeMatch.js'; + +type RealStandingRec = { + pk: 'REALSTANDING'; + sk: string; + standing: StandingPresetEntry[]; +}; + +export async function pauseMatchingRealStandingEntries( + client: DynamoDBDocumentClient, + tableName: string, + issuerId: string, + challenge: ChallengeForStandingMatch, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'REALSTANDING', sk: issuerId }, + })); + if (data.Item === undefined) { + return 0; + } + const rec = data.Item as RealStandingRec; + let paused = 0; + const standing = rec.standing.map(entry => { + if (challengeMatchesStandingEntry(challenge, entry)) { + if (!entry.suspended) { + paused += 1; + } + return { ...entry, suspended: true }; + } + return entry; + }); + if (paused === 0) { + return 0; + } + await client.send(new PutCommand({ + TableName: tableName, + Item: { + pk: 'REALSTANDING', + sk: issuerId, + standing, + }, + })); + return paused; +} diff --git a/crons/src/lib/pushSubscriptions.ts b/crons/src/lib/pushSubscriptions.ts new file mode 100644 index 00000000..787c2019 --- /dev/null +++ b/crons/src/lib/pushSubscriptions.ts @@ -0,0 +1,152 @@ +import { createHash } from 'crypto'; +import { + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + QueryCommand, +} from '@aws-sdk/lib-dynamodb'; +import webpush, { type RequestOptions } from 'web-push'; + +export type PushCredentials = { + pk: string; + sk: string; + payload: unknown; + endpoint?: string; + updatedAt?: string; +}; + +export type PushOptions = { + userId: string; + title: string; + body: string; + topic: 'yourturn' | 'ended' | 'started' | 'challenges' | 'test' | 'tournament'; + url?: string; +}; + +const PUSH_PK = 'PUSH'; +const PERMANENT_FAILURES = new Set([404, 410]); + +export function pushSubscriptionKey(endpoint: string): string { + return createHash('sha256').update(endpoint).digest('hex').slice(0, 16); +} + +export function pushSortKey(userId: string, endpoint: string): string { + return `${userId}#${pushSubscriptionKey(endpoint)}`; +} + +export async function queryPushSubscriptions( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, +): Promise { + const subscriptions: PushCredentials[] = []; + const skPrefix = `${userId}#`; + let lastKey: Record | undefined; + + do { + const result = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk AND begins_with(#sk, :skPrefix)', + ExpressionAttributeNames: { '#pk': 'pk', '#sk': 'sk' }, + ExpressionAttributeValues: { + ':pk': PUSH_PK, + ':skPrefix': skPrefix, + }, + ExclusiveStartKey: lastKey, + })); + if (result.Items !== undefined) { + subscriptions.push(...(result.Items as PushCredentials[])); + } + lastKey = result.LastEvaluatedKey; + } while (lastKey !== undefined); + + const legacy = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: PUSH_PK, sk: userId }, + })); + if (legacy.Item !== undefined) { + subscriptions.push(legacy.Item as PushCredentials); + } + + return subscriptions; +} + +async function deletePushSubscription( + client: DynamoDBDocumentClient, + tableName: string, + sk: string, +): Promise { + await client.send(new DeleteCommand({ + TableName: tableName, + Key: { pk: PUSH_PK, sk }, + })); +} + +export type SendNotificationFn = ( + subscription: unknown, + payload: string, + options: RequestOptions, +) => Promise; + +export async function sendPushToSubscriptions( + client: DynamoDBDocumentClient, + tableName: string, + opts: PushOptions, + subscriptions: PushCredentials[], + sendNotification: SendNotificationFn = webpush.sendNotification.bind(webpush), + logError: (err: unknown) => void = console.error, +): Promise { + if (subscriptions.length === 0) { + return; + } + + let subject = 'https://play.abstractplay.com'; + if (process.env.WEBSOCKET_STAGE === 'dev') { + subject = 'https://play.dev.abstractplay.com'; + } + + const { body, title, topic, url } = opts; + const options: RequestOptions = { + vapidDetails: { + subject, + publicKey: process.env.VAPID_PUBLIC_KEY as string, + privateKey: process.env.VAPID_PRIVATE_KEY as string, + }, + // @ts-expect-error web-push topic option + topic, + }; + const payload = JSON.stringify({ title, body, url, topic }); + + await Promise.allSettled( + subscriptions.map(async (sub) => { + try { + await sendNotification(sub.payload, payload, options); + } catch (err: unknown) { + const statusCode = typeof err === 'object' && err !== null && 'statusCode' in err + ? (err as { statusCode: number }).statusCode + : undefined; + if (statusCode !== undefined && PERMANENT_FAILURES.has(statusCode)) { + console.log(`Removing stale push subscription ${sub.sk} (${statusCode})`); + await deletePushSubscription(client, tableName, sub.sk); + } else { + logError(err); + } + } + }), + ); +} + +export async function sendPush( + client: DynamoDBDocumentClient, + tableName: string, + opts: PushOptions, +): Promise { + const publicKey = process.env.VAPID_PUBLIC_KEY; + const privateKey = process.env.VAPID_PRIVATE_KEY; + if (publicKey === undefined || publicKey === '' || privateKey === undefined || privateKey === '') { + console.log('VAPID keys not configured; skipping push'); + return; + } + const subscriptions = await queryPushSubscriptions(client, tableName, opts.userId); + await sendPushToSubscriptions(client, tableName, opts, subscriptions); +} diff --git a/crons/src/lib/ratingChangeNotifications.test.ts b/crons/src/lib/ratingChangeNotifications.test.ts new file mode 100644 index 00000000..b7751e5b --- /dev/null +++ b/crons/src/lib/ratingChangeNotifications.test.ts @@ -0,0 +1,327 @@ +import { describe, expect, it } from "vitest"; +import type { GlickoMeta } from "types/stats/GlickoStats.js"; +import type { StatSummaryRatings } from "types/stats/StatSummaryTiers.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; +import { GLICKO_MIN_GAMES_PROVISIONAL } from "../functions/summarizeHelpers.js"; +import { + buildRatingChangeSnapshot, + diffRatingChanges, + filterCandidates, + filterCandidatesByInAppPrefs, + MIN_RATING_DELTA, + RATINGS_NOTIFICATION_BASELINE, + toNotificationItems, + type RatingNotificationSnapshot, +} from "./ratingChangeNotifications.js"; + +function glickoRow( + user: string, + game: string, + ratingLow: number, + n: number, + provisional = false, + rd = 60, +): UserGameRating { + return { + user, + game, + rating: ratingLow + 100, + wld: [n, 0, 0], + glicko: { + rating: ratingLow + 100, + rd, + volatility: 0.06, + ratingLow, + ratingHigh: ratingLow + 200, + provisional, + established: !provisional, + n, + }, + }; +} + +function snapshotEntry( + ratingLow: number, + n: number, + rd = 60, + provisional = false, +): { ratingLow: number; rd: number; provisional: boolean; n: number } { + return { ratingLow, rd, provisional, n }; +} + +function minimalGlickoMeta(): GlickoMeta { + return { + establishedRd: 110, + provisionalRd: 200, + minGamesEstablished: 20, + minGamesProvisional: GLICKO_MIN_GAMES_PROVISIONAL, + periodMs: 86400000, + generatedAt: "2026-08-25T06:00:00.000Z", + counts: { + byGame: [], + site: { rated: 0, provisional: 0, established: 0 }, + }, + }; +} + +function ratingsSummary( + highest: UserGameRating[], + generatedAt = "2026-08-25T06:00:00.000Z", +): StatSummaryRatings { + return { + generated: generatedAt, + tier: "ratings", + ratings: { + highest, + avg: [], + weighted: [], + glickoByGame: [], + glickoSite: [], + glickoMeta: { ...minimalGlickoMeta(), generatedAt }, + playerCountsByUid: {}, + }, + }; +} + +const constants = { + minRatingDelta: MIN_RATING_DELTA, + minGamesProvisional: GLICKO_MIN_GAMES_PROVISIONAL, +}; + +describe("buildRatingChangeSnapshot", () => { + it("indexes highest rows by user and game label", () => { + const summary = ratingsSummary([ + glickoRow("alice", "chess (no variants)", 1200, 10), + glickoRow("alice", "go (9x9|handicap)", 1170, 5), + ]); + const snapshot = buildRatingChangeSnapshot(summary, summary.ratings.glickoMeta.generatedAt); + expect(snapshot.baseline).toBe(RATINGS_NOTIFICATION_BASELINE); + expect(snapshot.entries["alice|chess (no variants)"]).toEqual( + snapshotEntry(1200, 10), + ); + expect(snapshot.entries["alice|go (9x9|handicap)"]).toEqual( + snapshotEntry(1170, 5), + ); + }); +}); + +describe("filterCandidates gates", () => { + const prev: RatingNotificationSnapshot = { + generatedAt: "2026-08-24T06:20:00.000Z", + summaryGeneratedAt: "2026-08-24T06:00:00.000Z", + baseline: RATINGS_NOTIFICATION_BASELINE, + entries: { + "alice|chess (no variants)": snapshotEntry(1190, 10, 80), + "bob|chess (no variants)": snapshotEntry(1100, 8), + "carol|chess (no variants)": snapshotEntry(1000, 3, 120, true), + "dave|chess (no variants)": snapshotEntry(900, 2), + "eve|chess (no variants)": snapshotEntry(800, 15), + "frank|chess (no variants)": snapshotEntry(700, 12), + "grace|chess (no variants)": snapshotEntry(600, 20), + "henry|chess (no variants)": snapshotEntry(500, 11), + "ivy|chess (no variants)": snapshotEntry(400, 16), + "jack|chess (no variants)": snapshotEntry(300, 17), + "kate|chess (no variants)": snapshotEntry(200, 18), + "leo|chess (no variants)": snapshotEntry(100, 19), + "mary|chess (no variants)": snapshotEntry(50, 20), + }, + }; + + it("skips when n unchanged", () => { + const diffRows = diffRatingChanges(prev, [ + glickoRow("alice", "chess (no variants)", 1250, 10), + ]); + const { candidates, stats } = filterCandidates(diffRows, new Set(), constants); + expect(candidates).toHaveLength(0); + expect(stats.skippedNoActivity).toBe(1); + }); + + it("skips when delta below threshold", () => { + const diffRows = diffRatingChanges(prev, [ + glickoRow("alice", "chess (no variants)", 1193, 11), + ]); + const { candidates, stats } = filterCandidates(diffRows, new Set(), constants); + expect(candidates).toHaveLength(0); + expect(stats.skippedBelowThreshold).toBe(1); + }); + + it("notifies when n increased and delta meets threshold", () => { + const diffRows = diffRatingChanges(prev, [ + glickoRow("alice", "chess (no variants)", 1200, 11), + ]); + const { candidates } = filterCandidates(diffRows, new Set(), constants); + expect(candidates).toHaveLength(1); + expect(candidates[0]).toMatchObject({ + userId: "alice", + metaGameUid: "chess", + variants: [], + oldRating: 1190, + newRating: 1200, + oldRd: 80, + newRd: 60, + oldProvisional: false, + newProvisional: false, + delta: 10, + }); + }); + + it("skips provisional players below minGamesProvisional", () => { + const diffRows = diffRatingChanges(prev, [ + glickoRow("carol", "chess (no variants)", 1100, 4, true), + ]); + const { candidates, stats } = filterCandidates(diffRows, new Set(), constants); + expect(candidates).toHaveLength(0); + expect(stats.skippedProvisional).toBe(1); + }); + + it("skips bot users", () => { + const diffRows = diffRatingChanges(prev, [ + glickoRow("alice", "chess (no variants)", 1200, 11), + ]); + const { candidates, stats } = filterCandidates(diffRows, new Set(["alice"]), constants); + expect(candidates).toHaveLength(0); + expect(stats.skippedBot).toBe(1); + }); + + it("emits one notification for multiple games in same pool (aggregate delta)", () => { + const diffRows = diffRatingChanges(prev, [ + glickoRow("alice", "chess (no variants)", 1210, 15), + ]); + const { candidates } = filterCandidates(diffRows, new Set(), constants); + expect(candidates).toHaveLength(1); + expect(candidates[0]?.delta).toBe(20); + }); + + it("emits notifications for all qualifying different pools (no per-user cap)", () => { + const multiPrev: RatingNotificationSnapshot = { + generatedAt: "2026-08-24T06:20:00.000Z", + summaryGeneratedAt: "2026-08-24T06:00:00.000Z", + baseline: RATINGS_NOTIFICATION_BASELINE, + entries: { + "alice|chess (no variants)": snapshotEntry(1000, 5), + "alice|go (9x9|handicap)": snapshotEntry(1000, 5), + "alice|go (19x19)": snapshotEntry(1000, 5), + "alice|shogi (no variants)": snapshotEntry(1000, 5), + "alice|xiangqi (no variants)": snapshotEntry(1000, 5), + }, + }; + const diffRows = diffRatingChanges(multiPrev, [ + glickoRow("alice", "chess (no variants)", 1020, 6), + glickoRow("alice", "go (9x9|handicap)", 1020, 6), + glickoRow("alice", "go (19x19)", 1020, 6), + glickoRow("alice", "shogi (no variants)", 1020, 6), + glickoRow("alice", "xiangqi (no variants)", 1020, 6), + ]); + const { candidates } = filterCandidates(diffRows, new Set(), constants); + expect(candidates).toHaveLength(5); + }); +}); + +describe("filterCandidatesByInAppPrefs", () => { + const candidate = { + userId: "alice", + gameLabel: "chess (no variants)", + metaGameUid: "chess", + variants: [] as string[], + oldRating: 1000, + newRating: 1020, + oldRd: 80, + newRd: 70, + oldProvisional: false, + newProvisional: false, + delta: 20, + }; + + it("skips users who disabled ratingChange in-app notifications", () => { + const settings = new Map([ + ["alice", { all: { inAppNotifications: { ratingChange: false } } }], + ]); + const { candidates, skippedInAppPrefs } = filterCandidatesByInAppPrefs( + [candidate], + settings, + ); + expect(candidates).toHaveLength(0); + expect(skippedInAppPrefs).toBe(1); + }); + + it("keeps users with default or enabled prefs", () => { + const { candidates, skippedInAppPrefs } = filterCandidatesByInAppPrefs( + [candidate], + new Map(), + ); + expect(candidates).toHaveLength(1); + expect(skippedInAppPrefs).toBe(0); + }); +}); + +describe("toNotificationItems", () => { + it("matches node-backend ratingChange body shape", () => { + const items = toNotificationItems([ + { + userId: "alice", + gameLabel: "go (9x9|handicap)", + metaGameUid: "go", + variants: ["9x9", "handicap"], + oldRating: 1100, + newRating: 1120, + oldRd: 90, + newRd: 70, + oldProvisional: true, + newProvisional: false, + delta: 20, + }, + ], 1_700_000_000_000); + expect(items).toHaveLength(1); + expect(items[0]?.pk).toBe("NOTIFICATION#alice"); + expect(items[0]?.body).toEqual({ + type: "ratingChange", + metaGame: "go", + variants: ["9x9", "handicap"], + gameId: "", + oldRating: 1100, + newRating: 1120, + oldRd: 90, + newRd: 70, + oldProvisional: true, + newProvisional: false, + delta: 20, + }); + expect(items[0]?.expiresAt).toBeGreaterThan(1_700_000_000); + }); +}); + +describe("first run and idempotent re-run", () => { + it("first run builds snapshot with no prior entries to diff against", () => { + const summary = ratingsSummary([glickoRow("alice", "chess (no variants)", 1200, 1)]); + const snapshot = buildRatingChangeSnapshot(summary, summary.ratings.glickoMeta.generatedAt); + expect(Object.keys(snapshot.entries)).toHaveLength(1); + expect(snapshot.baseline).toBe(RATINGS_NOTIFICATION_BASELINE); + expect(snapshot.summaryGeneratedAt).toBe("2026-08-25T06:00:00.000Z"); + }); + + it("re-seeds when prior snapshot baseline is stale", () => { + const generatedAt = "2026-08-25T06:00:00.000Z"; + const stale: RatingNotificationSnapshot = { + generatedAt: "2026-08-24T06:20:00.000Z", + summaryGeneratedAt: "2026-08-24T06:00:00.000Z", + baseline: RATINGS_NOTIFICATION_BASELINE - 1, + entries: { + "alice|chess (no variants)": snapshotEntry(900, 5), + }, + }; + const nextSummary = ratingsSummary([glickoRow("alice", "chess (no variants)", 1250, 12)], generatedAt); + const diff = diffRatingChanges(stale, nextSummary.ratings.highest); + expect(diff[0]?.delta).toBeGreaterThan(100); + expect(stale.baseline).not.toBe(RATINGS_NOTIFICATION_BASELINE); + }); + + it("idempotent when summaryGeneratedAt unchanged", () => { + const generatedAt = "2026-08-25T06:00:00.000Z"; + const prev = buildRatingChangeSnapshot( + ratingsSummary([glickoRow("alice", "chess (no variants)", 1200, 11)], generatedAt), + generatedAt, + ); + const nextSummary = ratingsSummary([glickoRow("alice", "chess (no variants)", 1250, 12)], generatedAt); + expect(nextSummary.ratings.glickoMeta.generatedAt).toBe(prev.summaryGeneratedAt); + }); +}); diff --git a/crons/src/lib/ratingChangeNotifications.ts b/crons/src/lib/ratingChangeNotifications.ts new file mode 100644 index 00000000..233f47e9 --- /dev/null +++ b/crons/src/lib/ratingChangeNotifications.ts @@ -0,0 +1,279 @@ +import type { GlickoMeta } from "types/stats/GlickoStats.js"; +import type { StatSummaryRatings } from "types/stats/StatSummaryTiers.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; +import { RATINGS_NOTIFICATION_SNAPSHOT_KEY } from "../constants/recordsBucket.js"; +import { GLICKO_PRIOR_RATING_LOW, defaultGlickoPrior, parseBatchRatingGameLabel } from "./batchRatings.js"; +import { + wantsInAppNotification, + type InAppNotificationUserSettings, +} from "./inAppNotificationPrefs.js"; + +export const MIN_RATING_DELTA = 5; +export const NOTIFICATION_INITIAL_TTL_DAYS = 180; +/** Bump when rating methodology changes so the next notification run re-seeds without notifying. */ +export const RATINGS_NOTIFICATION_BASELINE = 2; +const SEC_PER_DAY = 86_400; +const NOTIFICATION_PK_PREFIX = "NOTIFICATION#"; + +export type RatingNotificationSnapshotEntry = { + ratingLow: number; + rd: number; + provisional: boolean; + n: number; +}; + +export type RatingNotificationSnapshot = { + generatedAt: string; + summaryGeneratedAt: string; + baseline: number; + entries: Record; +}; + +export type RatingChangeCandidate = { + userId: string; + gameLabel: string; + metaGameUid: string; + variants: string[]; + oldRating: number; + newRating: number; + oldRd: number; + newRd: number; + oldProvisional: boolean; + newProvisional: boolean; + delta: number; +}; + +type RatingChangeDiffRow = RatingChangeCandidate & { + oldN: number; + newN: number; +}; + +export type RatingChangeFilterStats = { + skippedNoActivity: number; + skippedBelowThreshold: number; + skippedProvisional: number; + skippedBot: number; + skippedInAppPrefs: number; +}; + +export type RatingChangeNotificationItem = { + pk: string; + sk: string; + body: { + type: "ratingChange"; + metaGame: string; + variants: string[]; + gameId: string; + oldRating: number; + newRating: number; + oldRd: number; + newRd: number; + oldProvisional: boolean; + newProvisional: boolean; + delta: number; + }; + expiresAt: number; +}; + +export type RatingChangeConstants = { + minRatingDelta: number; + minGamesProvisional: number; +}; + +function snapshotEntryKey(userId: string, gameLabel: string): string { + return `${userId}|${gameLabel}`; +} + +function roundRatingLow(ratingLow: number): number { + return Math.round(ratingLow); +} + +function roundRd(rd: number): number { + return Math.round(rd); +} + +const GLICKO_PRIOR = defaultGlickoPrior(); + +function uniqueSortKey(now = Date.now()): string { + return `${now}#${Math.random().toString(36).slice(2, 10)}`; +} + +function notificationExpiresAt(now = Date.now()): number { + return Math.floor(now / 1000) + NOTIFICATION_INITIAL_TTL_DAYS * SEC_PER_DAY; +} + +export function ratingChangeConstantsFromEnv( + glickoMeta: GlickoMeta, +): RatingChangeConstants { + const envDelta = process.env.MIN_RATING_DELTA; + const minRatingDelta = envDelta !== undefined && envDelta !== "" + ? Number(envDelta) + : MIN_RATING_DELTA; + return { + minRatingDelta: Number.isFinite(minRatingDelta) ? minRatingDelta : MIN_RATING_DELTA, + minGamesProvisional: glickoMeta.minGamesProvisional, + }; +} + +export function buildRatingChangeSnapshot( + summary: StatSummaryRatings, + summaryGeneratedAt: string, + generatedAt = new Date().toISOString(), +): RatingNotificationSnapshot { + const entries: Record = {}; + for (const row of summary.ratings.highest) { + const glicko = row.glicko; + if (glicko === undefined) { + continue; + } + entries[snapshotEntryKey(row.user, row.game)] = { + ratingLow: glicko.ratingLow, + rd: glicko.rd, + provisional: glicko.provisional, + n: glicko.n, + }; + } + return { + generatedAt, + summaryGeneratedAt, + baseline: RATINGS_NOTIFICATION_BASELINE, + entries, + }; +} + +export function diffRatingChanges( + prev: RatingNotificationSnapshot, + highest: UserGameRating[], +): RatingChangeDiffRow[] { + const rows: RatingChangeDiffRow[] = []; + for (const row of highest) { + const glicko = row.glicko; + if (glicko === undefined) { + continue; + } + const key = snapshotEntryKey(row.user, row.game); + const oldEntry = prev.entries[key]; + const oldRatingLow = oldEntry?.ratingLow ?? GLICKO_PRIOR_RATING_LOW; + const oldRd = oldEntry?.rd ?? GLICKO_PRIOR.rd; + const oldProvisional = oldEntry?.provisional ?? GLICKO_PRIOR.provisional; + const oldN = oldEntry?.n ?? 0; + const newRating = roundRatingLow(glicko.ratingLow); + const oldRating = roundRatingLow(oldRatingLow); + const newRd = roundRd(glicko.rd); + const oldRdRounded = roundRd(oldRd); + const { metaUid, variantUids } = parseBatchRatingGameLabel(row.game); + rows.push({ + userId: row.user, + gameLabel: row.game, + metaGameUid: metaUid, + variants: variantUids, + oldRating, + newRating, + oldRd: oldRdRounded, + newRd, + oldProvisional, + newProvisional: glicko.provisional, + delta: newRating - oldRating, + oldN, + newN: glicko.n, + }); + } + return rows; +} + +export function filterCandidates( + diffRows: RatingChangeDiffRow[], + botIds: Set, + constants: RatingChangeConstants, +): { candidates: RatingChangeCandidate[]; stats: RatingChangeFilterStats } { + const stats: RatingChangeFilterStats = { + skippedNoActivity: 0, + skippedBelowThreshold: 0, + skippedProvisional: 0, + skippedBot: 0, + skippedInAppPrefs: 0, + }; + const filtered: RatingChangeCandidate[] = []; + + for (const row of diffRows) { + if (botIds.has(row.userId)) { + stats.skippedBot += 1; + continue; + } + if (row.newN <= row.oldN) { + stats.skippedNoActivity += 1; + continue; + } + if (row.newProvisional && row.newN < constants.minGamesProvisional) { + stats.skippedProvisional += 1; + continue; + } + if (Math.abs(row.delta) < constants.minRatingDelta) { + stats.skippedBelowThreshold += 1; + continue; + } + filtered.push({ + userId: row.userId, + gameLabel: row.gameLabel, + metaGameUid: row.metaGameUid, + variants: row.variants, + oldRating: row.oldRating, + newRating: row.newRating, + oldRd: row.oldRd, + newRd: row.newRd, + oldProvisional: row.oldProvisional, + newProvisional: row.newProvisional, + delta: row.delta, + }); + } + + return { candidates: filtered, stats }; +} + +export function filterCandidatesByInAppPrefs( + candidates: RatingChangeCandidate[], + userSettings: ReadonlyMap, +): { candidates: RatingChangeCandidate[]; skippedInAppPrefs: number } { + const filtered: RatingChangeCandidate[] = []; + let skippedInAppPrefs = 0; + + for (const candidate of candidates) { + const settings = userSettings.get(candidate.userId); + if (!wantsInAppNotification(settings, "ratingChange")) { + skippedInAppPrefs += 1; + continue; + } + filtered.push(candidate); + } + + return { candidates: filtered, skippedInAppPrefs }; +} + +export function toNotificationItems( + candidates: RatingChangeCandidate[], + now = Date.now(), +): RatingChangeNotificationItem[] { + return candidates.map((candidate, index) => { + const itemNow = now + index; + return { + pk: `${NOTIFICATION_PK_PREFIX}${candidate.userId}`, + sk: uniqueSortKey(itemNow), + body: { + type: "ratingChange", + metaGame: candidate.metaGameUid, + variants: candidate.variants, + gameId: "", + oldRating: candidate.oldRating, + newRating: candidate.newRating, + oldRd: candidate.oldRd, + newRd: candidate.newRd, + oldProvisional: candidate.oldProvisional, + newProvisional: candidate.newProvisional, + delta: candidate.delta, + }, + expiresAt: notificationExpiresAt(itemNow), + }; + }); +} + +export { RATINGS_NOTIFICATION_SNAPSHOT_KEY }; diff --git a/crons/src/lib/revokeChallenge.test.ts b/crons/src/lib/revokeChallenge.test.ts new file mode 100644 index 00000000..1390f56f --- /dev/null +++ b/crons/src/lib/revokeChallenge.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest'; +import { + DeleteCommand, + GetCommand, + UpdateCommand, + type DynamoDBDocumentClient, +} from '@aws-sdk/lib-dynamodb'; +import { revokeChallengeRecord } from './revokeChallenge.js'; + +type StoredCommand = GetCommand | UpdateCommand | DeleteCommand; + +function mockClient( + botIds: Set = new Set(), +): DynamoDBDocumentClient & { commands: StoredCommand[] } { + const commands: StoredCommand[] = []; + return { + commands, + async send(command: StoredCommand) { + commands.push(command); + if (command instanceof GetCommand) { + const sk = command.input.Key?.sk; + if (command.input.Key?.pk === 'BOT' && typeof sk === 'string' && botIds.has(sk)) { + return { Item: { pk: 'BOT', sk } }; + } + return {}; + } + return {}; + }, + } as DynamoDBDocumentClient & { commands: StoredCommand[] }; +} + +describe('revokeChallengeRecord', () => { + it('deletes standing challenge and updates issuer and counts', async () => { + const client = mockClient(); + await revokeChallengeRecord(client, 'table', { + id: 'c1', + metaGame: 'chess', + numPlayers: 2, + challenger: { id: 'u1' }, + players: [{ id: 'u1' }, { id: 'u2' }], + }, true); + + const deletes = client.commands.filter(c => c instanceof DeleteCommand); + expect(deletes.some(c => c.input.Key?.pk === 'STANDINGCHALLENGE#chess')).toBe(true); + expect(client.commands.some(c => + c instanceof UpdateCommand + && c.input.UpdateExpression?.includes('challenges_standing'), + )).toBe(true); + expect(client.commands.some(c => + c instanceof UpdateCommand + && c.input.UpdateExpression?.includes('challenges_accepted'), + )).toBe(true); + }); + + it('deletes direct challenge and updates issued/received sets', async () => { + const client = mockClient(); + await revokeChallengeRecord(client, 'table', { + id: 'd1', + metaGame: 'go', + numPlayers: 2, + challenger: { id: 'u1' }, + challengees: [{ id: 'u2' }], + players: [{ id: 'u1' }], + }, false); + + expect(client.commands.some(c => + c instanceof DeleteCommand && c.input.Key?.pk === 'CHALLENGE', + )).toBe(true); + expect(client.commands.some(c => + c instanceof UpdateCommand + && c.input.UpdateExpression?.includes('challenges_issued'), + )).toBe(true); + expect(client.commands.some(c => + c instanceof UpdateCommand + && c.input.UpdateExpression?.includes('challenges_received'), + )).toBe(true); + }); +}); diff --git a/crons/src/lib/revokeChallenge.ts b/crons/src/lib/revokeChallenge.ts new file mode 100644 index 00000000..1057b0ba --- /dev/null +++ b/crons/src/lib/revokeChallenge.ts @@ -0,0 +1,91 @@ +/** + * Keep in sync with node-backend removeAChallenge (api/abstractplay.ts) for revocation paths. + */ +import { + DeleteCommand, + DynamoDBDocumentClient, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; +import { adjustShardedCounts } from './shardedMetaGameCounts.js'; +import { isBotId, isValidUserId } from './inactiveChallengeDiscovery.js'; + +export type ChallengePlayer = { id: string; name?: string }; + +export type RevokeChallengeRecord = { + id: string; + metaGame: string; + numPlayers: number; + challenger: ChallengePlayer; + challengees?: ChallengePlayer[]; + players?: ChallengePlayer[]; +}; + +export async function revokeChallengeRecord( + client: DynamoDBDocumentClient, + tableName: string, + challenge: RevokeChallengeRecord, + standing: boolean, +): Promise { + const work: Promise[] = []; + + if (!standing) { + work.push(client.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: challenge.challenger.id }, + UpdateExpression: 'DELETE challenges_issued :c', + ExpressionAttributeValues: { ':c': new Set([challenge.id]) }, + }))); + for (const challengee of challenge.challengees ?? []) { + if (!isValidUserId(challengee.id)) { + continue; + } + if (!(await isBotId(client, tableName, challengee.id))) { + work.push(client.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: challengee.id }, + UpdateExpression: 'DELETE challenges_received :c', + ExpressionAttributeValues: { ':c': new Set([challenge.id]) }, + }))); + } + } + } else { + work.push(client.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: challenge.challenger.id }, + UpdateExpression: 'DELETE challenges_standing :c', + ExpressionAttributeValues: { ':c': new Set([`${challenge.metaGame}#${challenge.id}`]) }, + }))); + } + + const acceptors = (challenge.players ?? []).filter( + p => isValidUserId(p.id) && p.id !== challenge.challenger.id, + ); + for (const player of acceptors) { + if (await isBotId(client, tableName, player.id)) { + continue; + } + work.push(client.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'USER', sk: player.id }, + UpdateExpression: 'DELETE challenges_accepted :c', + ExpressionAttributeValues: { + ':c': new Set([standing ? `${challenge.metaGame}#${challenge.id}` : challenge.id]), + }, + }))); + } + + if (!standing) { + work.push(client.send(new DeleteCommand({ + TableName: tableName, + Key: { pk: 'CHALLENGE', sk: challenge.id }, + }))); + } else { + work.push(client.send(new DeleteCommand({ + TableName: tableName, + Key: { pk: `STANDINGCHALLENGE#${challenge.metaGame}`, sk: challenge.id }, + }))); + work.push(adjustShardedCounts(client, tableName, challenge.metaGame, { standingchallenges: -1 })); + } + + await Promise.all(work); +} diff --git a/crons/src/lib/shardedMetaGameCounts.ts b/crons/src/lib/shardedMetaGameCounts.ts new file mode 100644 index 00000000..7bbc1b16 --- /dev/null +++ b/crons/src/lib/shardedMetaGameCounts.ts @@ -0,0 +1,71 @@ +import { + DynamoDBDocumentClient, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; + +export type ShardedCountDeltas = { + currentgames?: number; + completedgames?: number; + standingchallenges?: number; + stars?: number; + ratingsCount?: number; +}; + +export async function ensureShardedMetaGameCountEntry( + docClient: DynamoDBDocumentClient, + tableName: string, + metaGame: string, +): Promise { + await docClient.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: `METAGAMES#${metaGame}`, sk: 'COUNTS' }, + UpdateExpression: [ + 'SET currentgames = if_not_exists(currentgames, :z)', + 'completedgames = if_not_exists(completedgames, :z)', + 'standingchallenges = if_not_exists(standingchallenges, :z)', + 'stars = if_not_exists(stars, :z)', + 'ratingsCount = if_not_exists(ratingsCount, :z)', + ].join(', '), + ExpressionAttributeValues: { ':z': 0 }, + })); +} + +export async function adjustShardedCounts( + docClient: DynamoDBDocumentClient, + tableName: string, + metaGame: string, + deltas: ShardedCountDeltas, +): Promise { + await ensureShardedMetaGameCountEntry(docClient, tableName, metaGame); + const parts: string[] = []; + const values: Record = { ':z': 0 }; + if (deltas.currentgames !== undefined) { + parts.push('currentgames = if_not_exists(currentgames, :z) + :cg'); + values[':cg'] = deltas.currentgames; + } + if (deltas.completedgames !== undefined) { + parts.push('completedgames = if_not_exists(completedgames, :z) + :cd'); + values[':cd'] = deltas.completedgames; + } + if (deltas.standingchallenges !== undefined) { + parts.push('standingchallenges = if_not_exists(standingchallenges, :z) + :sc'); + values[':sc'] = deltas.standingchallenges; + } + if (deltas.stars !== undefined) { + parts.push('stars = if_not_exists(stars, :z) + :st'); + values[':st'] = deltas.stars; + } + if (deltas.ratingsCount !== undefined) { + parts.push('ratingsCount = if_not_exists(ratingsCount, :z) + :rc'); + values[':rc'] = deltas.ratingsCount; + } + if (parts.length === 0) { + return; + } + await docClient.send(new UpdateCommand({ + TableName: tableName, + Key: { pk: `METAGAMES#${metaGame}`, sk: 'COUNTS' }, + UpdateExpression: `SET ${parts.join(', ')}`, + ExpressionAttributeValues: values, + })); +} diff --git a/crons/src/lib/standingChallengeMatch.test.ts b/crons/src/lib/standingChallengeMatch.test.ts new file mode 100644 index 00000000..ce4e5fcf --- /dev/null +++ b/crons/src/lib/standingChallengeMatch.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; +import { + challengeMatchesStandingEntry, + stringArraysEqual, + type StandingPresetEntry, +} from './standingChallengeMatch.js'; + +const baseEntry = (): StandingPresetEntry => ({ + id: 'preset-1', + metaGame: 'chess', + numPlayers: 2, + variants: ['standard'], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + limit: 1, + sensitivity: 'variants', + suspended: false, +}); + +describe('stringArraysEqual', () => { + it('matches regardless of order', () => { + expect(stringArraysEqual(['a', 'b'], ['b', 'a'])).toBe(true); + expect(stringArraysEqual(['a'], ['b'])).toBe(false); + }); +}); + +describe('challengeMatchesStandingEntry', () => { + it('requires exact variants when sensitivity is variants', () => { + const entry = baseEntry(); + expect(challengeMatchesStandingEntry({ + metaGame: 'chess', + numPlayers: 2, + variants: ['standard'], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + }, entry)).toBe(true); + expect(challengeMatchesStandingEntry({ + metaGame: 'chess', + numPlayers: 2, + variants: ['other'], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + }, entry)).toBe(false); + }); + + it('matches meta sensitivity without variant equality', () => { + const entry = { ...baseEntry(), sensitivity: 'meta' as const, variants: ['a'] }; + expect(challengeMatchesStandingEntry({ + metaGame: 'chess', + numPlayers: 2, + variants: ['b'], + clockStart: 600, + clockInc: 0, + clockMax: 0, + clockHard: false, + rated: true, + noExplore: false, + }, entry)).toBe(true); + }); +}); diff --git a/crons/src/lib/standingChallengeMatch.ts b/crons/src/lib/standingChallengeMatch.ts new file mode 100644 index 00000000..66e78131 --- /dev/null +++ b/crons/src/lib/standingChallengeMatch.ts @@ -0,0 +1,74 @@ +export type StandingPresetEntry = { + id: string; + metaGame: string; + numPlayers: number; + variants?: string[]; + clockStart: number; + clockInc: number; + clockMax: number; + clockHard: boolean; + rated: boolean; + noExplore?: boolean; + sensitivity: 'meta' | 'variants'; + suspended: boolean; +}; + +export type ChallengeForStandingMatch = { + metaGame: string; + numPlayers: number; + variants?: string[]; + clockStart: number; + clockInc: number; + clockMax: number; + clockHard: boolean; + rated: boolean; + noExplore?: boolean; +}; + +export function stringArraysEqual(lst1: string[], lst2: string[]): boolean { + if (lst1.length !== lst2.length) { + return false; + } + const s1 = [...lst1].sort((a, b) => a.localeCompare(b)); + const s2 = [...lst2].sort((a, b) => a.localeCompare(b)); + for (let i = 0; i < s1.length; i++) { + if (s1[i] !== s2[i]) { + return false; + } + } + return true; +} + +export function challengeMatchesStandingEntry( + challenge: ChallengeForStandingMatch, + entry: StandingPresetEntry, +): boolean { + if (challenge.metaGame !== entry.metaGame) { + return false; + } + if (challenge.numPlayers !== entry.numPlayers) { + return false; + } + if (challenge.clockStart !== entry.clockStart) { + return false; + } + if (challenge.clockInc !== entry.clockInc) { + return false; + } + if (challenge.clockMax !== entry.clockMax) { + return false; + } + if (challenge.clockHard !== entry.clockHard) { + return false; + } + if (challenge.rated !== entry.rated) { + return false; + } + if ((challenge.noExplore ?? false) !== (entry.noExplore ?? false)) { + return false; + } + if (entry.sensitivity === 'variants') { + return stringArraysEqual(challenge.variants ?? [], entry.variants ?? []); + } + return true; +} diff --git a/crons/src/lib/standingChallengeQueries.ts b/crons/src/lib/standingChallengeQueries.ts new file mode 100644 index 00000000..65310e6f --- /dev/null +++ b/crons/src/lib/standingChallengeQueries.ts @@ -0,0 +1,99 @@ +import { gameinfo } from '@abstractplay/gameslib'; +import { + BatchGetCommand, + QueryCommand, + type DynamoDBDocumentClient, +} from '@aws-sdk/lib-dynamodb'; + +export type OpenChallengeRecord = Record & { + id?: string; + metaGame: string; + challenger?: { id?: string; name?: string }; +}; + +function metaGameUids(): string[] { + const metaGames: string[] = []; + gameinfo.forEach(g => metaGames.push(g.uid)); + return metaGames; +} + +export async function listMetaGamesWithStandingChallenges( + client: DynamoDBDocumentClient, + tableName: string, +): Promise { + const metaGames = metaGameUids(); + const active: string[] = []; + for (let i = 0; i < metaGames.length; i += 100) { + const chunk = metaGames.slice(i, i + 100); + const data = await client.send(new BatchGetCommand({ + RequestItems: { + [tableName]: { + Keys: chunk.map(metaGame => ({ pk: `METAGAMES#${metaGame}`, sk: 'COUNTS' })), + }, + }, + })); + for (const item of data.Responses?.[tableName] ?? []) { + const metaGame = String(item.pk).replace('METAGAMES#', ''); + if (((item.standingchallenges as number | undefined) ?? 0) > 0) { + active.push(metaGame); + } + } + } + return active; +} + +export async function queryStandingChallengesForMetaGame( + client: DynamoDBDocumentClient, + tableName: string, + metaGame: string, +): Promise { + const items: OpenChallengeRecord[] = []; + let lastKey: Record | undefined; + do { + const result = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + ExpressionAttributeValues: { ':pk': `STANDINGCHALLENGE#${metaGame}` }, + ExclusiveStartKey: lastKey, + })); + for (const item of result.Items ?? []) { + items.push(item as OpenChallengeRecord); + } + lastKey = result.LastEvaluatedKey; + } while (lastKey !== undefined); + return items; +} + +export async function queryAllStandingChallenges( + client: DynamoDBDocumentClient, + tableName: string, +): Promise { + const activeMetaGames = await listMetaGamesWithStandingChallenges(client, tableName); + const chunks = await Promise.all( + activeMetaGames.map(metaGame => queryStandingChallengesForMetaGame(client, tableName, metaGame)), + ); + return chunks.flat(); +} + +export async function queryAllDirectChallenges( + client: DynamoDBDocumentClient, + tableName: string, +): Promise { + const items: OpenChallengeRecord[] = []; + let lastKey: Record | undefined; + do { + const result = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + ExpressionAttributeValues: { ':pk': 'CHALLENGE' }, + ExclusiveStartKey: lastKey, + })); + for (const item of result.Items ?? []) { + items.push(item as OpenChallengeRecord); + } + lastKey = result.LastEvaluatedKey; + } while (lastKey !== undefined); + return items; +} diff --git a/crons/src/lib/tournamentGame.test.ts b/crons/src/lib/tournamentGame.test.ts new file mode 100644 index 00000000..b7bf8a17 --- /dev/null +++ b/crons/src/lib/tournamentGame.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; +import { gameinfo } from '@abstractplay/gameslib'; +import { tournamentPlaySupported } from './tournamentGame.js'; + +describe('tournamentPlaySupported', () => { + it('is false for unknown games', () => { + expect(tournamentPlaySupported('not-a-real-game-uid')).toBe(false); + }); + + it('is true when playercounts includes 2', () => { + const withTwo = [...gameinfo.values()].find((g) => g.playercounts.includes(2)); + if (withTwo !== undefined) { + expect(tournamentPlaySupported(withTwo.uid)).toBe(true); + } + }); + + it('is false when playercounts is solo-only', () => { + const soloOnly = [...gameinfo.values()].find( + (g) => g.playercounts.length === 1 && g.playercounts[0] === 1, + ); + if (soloOnly !== undefined) { + expect(tournamentPlaySupported(soloOnly.uid)).toBe(false); + } + }); +}); diff --git a/crons/src/lib/tournamentGame.ts b/crons/src/lib/tournamentGame.ts new file mode 100644 index 00000000..861b02c2 --- /dev/null +++ b/crons/src/lib/tournamentGame.ts @@ -0,0 +1,7 @@ +import { gameinfo } from '@abstractplay/gameslib'; + +/** Whether automated tournaments can run for this title (requires `playercount: 2`). */ +export const tournamentPlaySupported = (metaGame: string): boolean => { + const info = gameinfo.get(metaGame); + return info !== undefined && info.playercounts.includes(2); +}; diff --git a/crons/src/lib/tournamentPairing.test.ts b/crons/src/lib/tournamentPairing.test.ts new file mode 100644 index 00000000..453358d1 --- /dev/null +++ b/crons/src/lib/tournamentPairing.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { + canonicalPlayerPair, + existingPairKeys, + findExistingGameForPair, + type ExistingTournamentGame, +} from './tournamentPairing.js'; + +describe('tournamentPairing resume helpers', () => { + const games: ExistingTournamentGame[] = [ + { id: 'g1', division: 1, player1: 'alice', player2: 'bob', pairKey: 'alice#bob' }, + { id: 'g2', division: 1, player1: 'carol', player2: 'dave', pairKey: 'carol#dave' }, + ]; + + it('builds a set of existing pair keys', () => { + expect(existingPairKeys(games)).toEqual(new Set(['alice#bob', 'carol#dave'])); + }); + + it('finds an existing game for a canonical pair', () => { + expect(findExistingGameForPair(games, canonicalPlayerPair('bob', 'alice'))?.id).toBe('g1'); + expect(findExistingGameForPair(games, 'eve#frank')).toBeUndefined(); + }); +}); diff --git a/crons/src/lib/tournamentPairing.ts b/crons/src/lib/tournamentPairing.ts new file mode 100644 index 00000000..e8c71386 --- /dev/null +++ b/crons/src/lib/tournamentPairing.ts @@ -0,0 +1,104 @@ +import { + DynamoDBDocumentClient, + GetCommand, + PutCommand, + QueryCommand, +} from '@aws-sdk/lib-dynamodb'; + +export function canonicalPlayerPair(player1: string, player2: string): string { + return [player1, player2].sort((a, b) => a.localeCompare(b)).join('#'); +} + +export type ExistingTournamentGame = { + id: string; + division: number; + player1: string; + player2: string; + pairKey: string; +}; + +export async function loadExistingTournamentGames( + client: DynamoDBDocumentClient, + tableName: string, + tournamentId: string, +): Promise { + const found: ExistingTournamentGame[] = []; + let lastEvaluatedKey: Record | undefined; + + do { + const page = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk', + FilterExpression: '#t = :tid', + ExpressionAttributeNames: { '#pk': 'pk', '#t': 'tournament' }, + ExpressionAttributeValues: { ':pk': 'GAME', ':tid': tournamentId }, + ExclusiveStartKey: lastEvaluatedKey, + })); + + for (const item of page.Items ?? []) { + const game = item as { + id: string; + division?: number; + players?: { id: string }[]; + }; + if (game.players === undefined || game.players.length < 2) { + continue; + } + const player1 = game.players[0]!.id; + const player2 = game.players[1]!.id; + found.push({ + id: game.id, + division: game.division ?? 1, + player1, + player2, + pairKey: canonicalPlayerPair(player1, player2), + }); + } + lastEvaluatedKey = page.LastEvaluatedKey; + } while (lastEvaluatedKey); + + return found; +} + +export async function ensureTournamentGameLink( + client: DynamoDBDocumentClient, + tableName: string, + tournamentId: string, + division: number, + gameId: string, + player1: string, + player2: string, +): Promise { + const sk = `${tournamentId}#${division.toString()}#${gameId}`; + const existing = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'TOURNAMENTGAME', sk }, + ProjectionExpression: '#pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + })); + if (existing.Item !== undefined) { + return false; + } + await client.send(new PutCommand({ + TableName: tableName, + Item: { + pk: 'TOURNAMENTGAME', + sk, + id: gameId, + player1, + player2, + }, + })); + return true; +} + +export function existingPairKeys(games: ExistingTournamentGame[]): Set { + return new Set(games.map(g => g.pairKey)); +} + +export function findExistingGameForPair( + games: ExistingTournamentGame[], + pairKey: string, +): ExistingTournamentGame | undefined { + return games.find(g => g.pairKey === pairKey); +} diff --git a/crons/src/lib/tournamentStartNotifications.test.ts b/crons/src/lib/tournamentStartNotifications.test.ts new file mode 100644 index 00000000..096a828a --- /dev/null +++ b/crons/src/lib/tournamentStartNotifications.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { PutCommand, GetCommand } from "@aws-sdk/lib-dynamodb"; +import { enqueueTournamentStartNotifications } from "./tournamentStartNotifications.js"; + +type StoreItem = Record; + +function createMockClient(store: Map) { + return { + async send(command: unknown) { + if (command instanceof GetCommand) { + const key = command.input.Key as { pk: string; sk: string }; + if (key.pk === "BOT") { + return {}; + } + return {}; + } + if (command instanceof PutCommand) { + const item = command.input.Item as StoreItem; + const pk = item.pk as string; + const sk = item.sk as string; + store.set(`${pk}:${sk}`, item); + return {}; + } + throw new Error(`Unexpected command: ${String(command)}`); + }, + }; +} + +describe("enqueueTournamentStartNotifications", () => { + it("writes tournamentStart notifications for human players with default prefs", async () => { + const store = new Map(); + const client = createMockClient(store); + + await enqueueTournamentStartNotifications( + client as never, + "abstract-play-test", + { + id: "tour-1", + metaGame: "go", + number: 4, + variants: ["small"], + }, + ["user-a", "user-b"], + ); + + expect(store.size).toBe(2); + for (const item of store.values()) { + expect(item.body).toEqual({ + type: "tournamentStart", + tournamentId: "tour-1", + metaGame: "go", + number: 4, + variants: ["small"], + }); + } + }); + + it("skips players who disabled tournamentStart in-app notifications", async () => { + const store = new Map(); + const client = createMockClient(store); + const settingsByUserId = new Map([ + ["user-a", { all: { inAppNotifications: { tournamentStart: false } } }], + ["user-b", undefined], + ]); + + await enqueueTournamentStartNotifications( + client as never, + "abstract-play-test", + { + id: "tour-1", + metaGame: "go", + number: 4, + }, + ["user-a", "user-b"], + settingsByUserId, + ); + + expect(store.size).toBe(1); + const item = [...store.values()][0]; + expect((item.pk as string).endsWith("user-b")).toBe(true); + }); +}); diff --git a/crons/src/lib/tournamentStartNotifications.ts b/crons/src/lib/tournamentStartNotifications.ts new file mode 100644 index 00000000..be4c4b49 --- /dev/null +++ b/crons/src/lib/tournamentStartNotifications.ts @@ -0,0 +1,115 @@ +import { + DynamoDBDocumentClient, + GetCommand, + PutCommand, +} from '@aws-sdk/lib-dynamodb'; +import { + wantsInAppNotification, + type InAppNotificationUserSettings, +} from './inAppNotificationPrefs.js'; + +const NOTIFICATION_PK_PREFIX = 'NOTIFICATION#'; +const NOTIFICATION_INITIAL_TTL_DAYS = 180; +const SEC_PER_DAY = 86_400; + +export type TournamentStartNotificationTournament = { + id: string; + metaGame: string; + number: number; + variants?: string[]; +}; + +type TournamentStartNotificationBody = { + type: 'tournamentStart'; + tournamentId: string; + metaGame: string; + number: number; + variants: string[]; +}; + +function notificationPk(userId: string): string { + return `${NOTIFICATION_PK_PREFIX}${userId}`; +} + +function notificationInitialExpiresAt(now = Date.now()): number { + return Math.floor(now / 1000) + NOTIFICATION_INITIAL_TTL_DAYS * SEC_PER_DAY; +} + +function uniqueSortKey(now = Date.now()): string { + return `${now}#${Math.random().toString(36).slice(2, 10)}`; +} + +async function isBotId( + client: DynamoDBDocumentClient, + tableName: string, + id: string, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk: 'BOT', sk: id }, + ProjectionExpression: '#pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + })); + return data.Item !== undefined; +} + +async function filterHumanIds( + client: DynamoDBDocumentClient, + tableName: string, + ids: string[], +): Promise { + const human: string[] = []; + for (const id of ids) { + if (!(await isBotId(client, tableName, id))) { + human.push(id); + } + } + return human; +} + +async function createTournamentStartNotification( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, + body: TournamentStartNotificationBody, +): Promise { + if (await isBotId(client, tableName, userId)) { + return; + } + const now = Date.now(); + await client.send(new PutCommand({ + TableName: tableName, + Item: { + pk: notificationPk(userId), + sk: uniqueSortKey(now), + body, + expiresAt: notificationInitialExpiresAt(now), + }, + })); +} + +export async function enqueueTournamentStartNotifications( + client: DynamoDBDocumentClient, + tableName: string, + tournament: TournamentStartNotificationTournament, + playerIds: string[], + settingsByUserId?: ReadonlyMap, +): Promise { + const humanIds = await filterHumanIds(client, tableName, playerIds); + const variants = tournament.variants ?? []; + const bodyBase = { + type: 'tournamentStart' as const, + tournamentId: tournament.id, + metaGame: tournament.metaGame, + number: tournament.number, + variants, + }; + + await Promise.all(humanIds.map(async (userId) => { + const settings = settingsByUserId?.get(userId); + if (!wantsInAppNotification(settings, 'tournamentStart')) { + return; + } + await createTournamentStartNotification(client, tableName, userId, bodyBase); + })); +} diff --git a/crons/src/lib/writeJournal.test.ts b/crons/src/lib/writeJournal.test.ts new file mode 100644 index 00000000..87883b89 --- /dev/null +++ b/crons/src/lib/writeJournal.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { DeleteCommand, PutCommand } from '@aws-sdk/lib-dynamodb'; +import { WriteJournal } from './writeJournal.js'; + +describe('WriteJournal', () => { + it('rolls back creates in reverse order', async () => { + const journal = new WriteJournal(); + journal.trackCreate('GAME', 'chess#0#g1'); + journal.trackCreate('TOURNAMENTGAME', 't#1#g1'); + + const calls: string[] = []; + await journal.rollback( + {} as never, + 'table', + async (cmd) => { + if (cmd instanceof DeleteCommand) { + calls.push(`${cmd.input.Key?.pk}/${cmd.input.Key?.sk}`); + } + return {}; + }, + ); + + expect(calls).toEqual(['TOURNAMENTGAME/t#1#g1', 'GAME/chess#0#g1']); + }); + + it('restores replaced items on rollback', async () => { + const journal = new WriteJournal(); + journal.trackReplace({ pk: 'TOURNAMENT', sk: 't1', started: false }, 'TOURNAMENT', 't1'); + + const puts: Record[] = []; + await journal.rollback( + {} as never, + 'table', + async (cmd) => { + if (cmd instanceof PutCommand) { + puts.push(cmd.input.Item as Record); + } + return {}; + }, + ); + + expect(puts).toHaveLength(1); + expect(puts[0]?.started).toBe(false); + }); +}); + +describe('canonicalPlayerPair', () => { + it('orders player ids consistently', async () => { + const { canonicalPlayerPair } = await import('./tournamentPairing.js'); + expect(canonicalPlayerPair('b', 'a')).toBe('a#b'); + expect(canonicalPlayerPair('a', 'b')).toBe('a#b'); + }); +}); diff --git a/crons/src/lib/writeJournal.ts b/crons/src/lib/writeJournal.ts new file mode 100644 index 00000000..ec08bbb4 --- /dev/null +++ b/crons/src/lib/writeJournal.ts @@ -0,0 +1,114 @@ +import { + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + PutCommand, + UpdateCommand, +} from '@aws-sdk/lib-dynamodb'; + +export type DynamoItem = Record; + +export type RollbackEntry = + | { op: 'delete'; pk: string; sk: string } + | { op: 'put'; item: DynamoItem }; + +export class WriteJournal { + private readonly entries: RollbackEntry[] = []; + + /** Record that rollback should delete a row created during this attempt. */ + trackCreate(pk: string, sk: string): void { + this.entries.push({ op: 'delete', pk, sk }); + } + + /** Before overwrite/delete, snapshot prior row for rollback (or delete if absent). */ + trackReplace(previous: DynamoItem | undefined, pk: string, sk: string): void { + if (previous !== undefined) { + this.entries.push({ op: 'put', item: { ...previous } }); + } else { + this.entries.push({ op: 'delete', pk, sk }); + } + } + + get size(): number { + return this.entries.length; + } + + async rollback( + client: DynamoDBDocumentClient, + tableName: string, + send: (command: DeleteCommand | PutCommand) => Promise, + ): Promise { + for (let i = this.entries.length - 1; i >= 0; i--) { + const entry = this.entries[i]!; + try { + if (entry.op === 'delete') { + await send(new DeleteCommand({ + TableName: tableName, + Key: { pk: entry.pk, sk: entry.sk }, + })); + } else { + await send(new PutCommand({ + TableName: tableName, + Item: entry.item, + })); + } + } catch (err) { + console.error(`Rollback failed for ${entry.op} on ${entry.op === 'delete' ? `${entry.pk}/${entry.sk}` : entry.item.pk}/${entry.item.sk}:`, err); + } + } + } +} + +export async function loadItem( + client: DynamoDBDocumentClient, + tableName: string, + pk: string, + sk: string, +): Promise { + const data = await client.send(new GetCommand({ + TableName: tableName, + Key: { pk, sk }, + })); + return data.Item as DynamoItem | undefined; +} + +export async function acquireTournamentStartingLock( + client: DynamoDBDocumentClient, + tableName: string, + tournamentId: string, + send: (command: UpdateCommand) => Promise, +): Promise { + try { + await send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'TOURNAMENT', sk: tournamentId }, + ConditionExpression: 'started = :f AND attribute_not_exists(starting)', + UpdateExpression: 'SET starting = :t, startAttemptAt = :now', + ExpressionAttributeValues: { ':f': false, ':t': true, ':now': Date.now() }, + })); + return true; + } catch (err: unknown) { + if ((err as { name?: string }).name === 'ConditionalCheckFailedException') { + console.log(`Tournament ${tournamentId} is already starting or started`); + return false; + } + throw err; + } +} + +export async function releaseTournamentStartingLock( + client: DynamoDBDocumentClient, + tableName: string, + tournamentId: string, + send: (command: UpdateCommand) => Promise, +): Promise { + try { + await send(new UpdateCommand({ + TableName: tableName, + Key: { pk: 'TOURNAMENT', sk: tournamentId }, + UpdateExpression: 'REMOVE starting, startAttemptAt', + })); + } catch (err) { + console.error(`Failed to release starting lock on tournament ${tournamentId}:`, err); + } +} diff --git a/crons/src/locales/README.md b/crons/src/locales/README.md new file mode 100644 index 00000000..c1fa5952 --- /dev/null +++ b/crons/src/locales/README.md @@ -0,0 +1,9 @@ +# Generated `apback` locale files + +Not committed. Canonical strings live in [node-backend `locales/`](https://github.com/AbstractPlay/node-backend/tree/develop/locales). + +**Local:** `npm run sync-apback-locales` (sibling `../node-backend` or `NODE_BACKEND_ROOT`). + +**CI / deploy:** workflows check out node-backend and run the same sync before build and test. + +`npm test` runs `pretest`, which syncs automatically when these files are missing. diff --git a/crons/src/types/BasicRec.ts b/crons/src/types/BasicRec.ts new file mode 100644 index 00000000..291be169 --- /dev/null +++ b/crons/src/types/BasicRec.ts @@ -0,0 +1,7 @@ +export type BasicRec = { + Item: { + pk: string; + sk: string; + [key: string]: any; + } +} diff --git a/crons/src/types/GameRec.ts b/crons/src/types/GameRec.ts new file mode 100644 index 00000000..d83a0b2b --- /dev/null +++ b/crons/src/types/GameRec.ts @@ -0,0 +1,17 @@ +export type GameRec = { + pk: string; + sk: string; + id: string; + metaGame: string; + state: string; + pieInvoked?: boolean; + rated?: boolean; + players: { + name: string; + id: string; + time: number; + }[]; + tournament?: string; + event?: string; + [key: string]: any; +} diff --git a/crons/src/types/MoveRec.ts b/crons/src/types/MoveRec.ts new file mode 100644 index 00000000..ea71470d --- /dev/null +++ b/crons/src/types/MoveRec.ts @@ -0,0 +1,5 @@ +export type MoveRec = { + metaGame: string; + player: string; + time: number; +}; diff --git a/crons/src/types/OrgEvent.ts b/crons/src/types/OrgEvent.ts new file mode 100644 index 00000000..6d18fab4 --- /dev/null +++ b/crons/src/types/OrgEvent.ts @@ -0,0 +1,11 @@ +export type OrgEvent = { + pk: "ORGEVENT"; + sk: string; // + name: string; + description: string; + organizer: string; + dateStart: number; + dateEnd?: number; + winner?: string[]; + visible: boolean; +} diff --git a/crons/src/types/OrgEventGame.ts b/crons/src/types/OrgEventGame.ts new file mode 100644 index 00000000..f937bb9e --- /dev/null +++ b/crons/src/types/OrgEventGame.ts @@ -0,0 +1,12 @@ +export type OrgEventGame = { + pk: "ORGEVENTGAME"; + sk: string; // # + metaGame: string; + variants?: string[]; + round: number; + gameid: string; + player1: string; + player2: string; + winner?: number[]; + arbitrated?: boolean; +}; diff --git a/crons/src/types/Tournament.ts b/crons/src/types/Tournament.ts new file mode 100644 index 00000000..2ed1db1b --- /dev/null +++ b/crons/src/types/Tournament.ts @@ -0,0 +1,12 @@ +export type Tournament = { + pk: string; + sk: string; + id: string; + metaGame: string; + variants: string[]; + number: number; + started: boolean; + dateCreated: number; + datePreviousEnded: number; // 0 means either the first tournament or a restart of the series (after it stopped because not enough participants), 3000000000000 means previous tournament still running. + [key: string]: any; +}; diff --git a/crons/src/types/domain.ts b/crons/src/types/domain.ts new file mode 100644 index 00000000..e8f6323a --- /dev/null +++ b/crons/src/types/domain.ts @@ -0,0 +1,10 @@ +export interface User { + id: string; + name: string; + email: string; +} + +export interface StatusPayload { + status: string; + timestamp: string; +} diff --git a/crons/src/types/index.ts b/crons/src/types/index.ts new file mode 100644 index 00000000..4d4d9cdf --- /dev/null +++ b/crons/src/types/index.ts @@ -0,0 +1,7 @@ +export * from "./stats/index.js"; +export type { BasicRec } from "./BasicRec.js"; +export type { GameRec } from "./GameRec.js"; +export type { MoveRec } from "./MoveRec.js"; +export type { Tournament } from "./Tournament.js" +export type { OrgEvent } from "./OrgEvent.js"; +export type { OrgEventGame } from "./OrgEventGame.js"; diff --git a/crons/src/types/json.d.ts b/crons/src/types/json.d.ts new file mode 100644 index 00000000..f3f59664 --- /dev/null +++ b/crons/src/types/json.d.ts @@ -0,0 +1,4 @@ +declare module "*.json" { + const value: any; + export default value; +} diff --git a/crons/src/types/playerSummaryQueue.ts b/crons/src/types/playerSummaryQueue.ts new file mode 100644 index 00000000..0eac155b --- /dev/null +++ b/crons/src/types/playerSummaryQueue.ts @@ -0,0 +1,44 @@ +import type { PlayerSummarySlice } from "types/stats/StatSummaryTiers.js"; + +export type PlayerSummaryQueueMessage = { + user: string; + key: string; + slice: PlayerSummarySlice; +}; + +export type PlayerSummaryManifestV1 = { + version: 1; + generated: string; + expectedCount: number; + enqueuedAt: string; +}; + +export type PlayerSummaryManifest = { + version: 2; + generated: string; + enqueuedAt: string; + candidateCount: number; + expectedCount: number; + skippedCount: number; + inputFingerprint: string; + contentHashes: Record; +}; + +export type PlayerSummaryManifestAny = PlayerSummaryManifestV1 | PlayerSummaryManifest; + +export function parsePreviousPlayerSummaryManifest( + raw: unknown, +): { inputFingerprint?: string; contentHashes?: Record } { + if (raw === null || typeof raw !== "object") { + return {}; + } + const manifest = raw as Record; + if (manifest.version === 2) { + const v2 = manifest as PlayerSummaryManifest; + return { + inputFingerprint: v2.inputFingerprint, + contentHashes: v2.contentHashes, + }; + } + return {}; +} diff --git a/crons/src/types/stats/GameNumList.ts b/crons/src/types/stats/GameNumList.ts new file mode 100644 index 00000000..57b0765e --- /dev/null +++ b/crons/src/types/stats/GameNumList.ts @@ -0,0 +1,4 @@ +export interface GameNumList { + game: string; + value: number[]; +} diff --git a/crons/src/types/stats/GameNumber.ts b/crons/src/types/stats/GameNumber.ts new file mode 100644 index 00000000..7e60693b --- /dev/null +++ b/crons/src/types/stats/GameNumber.ts @@ -0,0 +1,4 @@ +export interface GameNumber { + game: string; + value: number; +} diff --git a/crons/src/types/stats/GeoStats.ts b/crons/src/types/stats/GeoStats.ts new file mode 100644 index 00000000..d3ecb59d --- /dev/null +++ b/crons/src/types/stats/GeoStats.ts @@ -0,0 +1,5 @@ +export interface GeoStats { + code: string; + name: string; + n: number; +} diff --git a/crons/src/types/stats/GlickoStats.ts b/crons/src/types/stats/GlickoStats.ts new file mode 100644 index 00000000..6680b606 --- /dev/null +++ b/crons/src/types/stats/GlickoStats.ts @@ -0,0 +1,53 @@ +export interface GlickoStats { + rating: number; + rd: number; + volatility: number; + ratingLow: number; + ratingHigh: number; + provisional: boolean; + established: boolean; + n: number; +} + +export type GlickoByGameRow = { + user: string; + game: string; + glicko: GlickoStats; +}; + +export type GlickoSiteEntry = { + user: string; + rating: number; + rd: number; + ratingLow: number; + ratingHigh: number; + n: number; + provisional: boolean; + established: boolean; +}; + +export type GlickoGameCounts = { + game: string; + rated: number; + provisional: number; + established: number; +}; + +export type GlickoSiteCounts = { + rated: number; + provisional: number; + established: number; +}; + +export type GlickoMeta = { + establishedRd: number; + provisionalRd: number; + minGamesEstablished: number; + minGamesProvisional: number; + periodMs: number; + generatedAt: string; + counts: { + byGame: GlickoGameCounts[]; + site: GlickoSiteCounts; + }; +}; diff --git a/crons/src/types/stats/HoursPerStats.ts b/crons/src/types/stats/HoursPerStats.ts new file mode 100644 index 00000000..261ae8bb --- /dev/null +++ b/crons/src/types/stats/HoursPerStats.ts @@ -0,0 +1,10 @@ +export type HoursPerStats = { + /** Move-weighted mean of winsorized per-game hours-per-move rates */ + mean: number; + /** Median of winsorized per-game hours-per-move rates */ + median: number; + /** Number of qualifying games */ + n: number; + /** Median winsorized hours per move per week bucket (aligned with histograms.all) */ + byWeek: number[]; +}; diff --git a/crons/src/types/stats/MetaPieStats.ts b/crons/src/types/stats/MetaPieStats.ts new file mode 100644 index 00000000..30627689 --- /dev/null +++ b/crons/src/types/stats/MetaPieStats.ts @@ -0,0 +1,6 @@ +export type MetaPieStats = { + game: string; + n: number; + pied: number; + rate: number; +}; diff --git a/crons/src/types/stats/MetaPlayerCountMix.ts b/crons/src/types/stats/MetaPlayerCountMix.ts new file mode 100644 index 00000000..47f52d41 --- /dev/null +++ b/crons/src/types/stats/MetaPlayerCountMix.ts @@ -0,0 +1,6 @@ +export type MetaPlayerCountMix = { + game: string; + byCount: { + [playerCount: string]: number; + }; +}; diff --git a/crons/src/types/stats/PlayContextStats.ts b/crons/src/types/stats/PlayContextStats.ts new file mode 100644 index 00000000..4e0f1843 --- /dev/null +++ b/crons/src/types/stats/PlayContextStats.ts @@ -0,0 +1,4 @@ +export type PlayContextStats = { + casual: number; + event: number; +}; diff --git a/crons/src/types/stats/PlayerTimeoutStats.ts b/crons/src/types/stats/PlayerTimeoutStats.ts new file mode 100644 index 00000000..31c21641 --- /dev/null +++ b/crons/src/types/stats/PlayerTimeoutStats.ts @@ -0,0 +1,5 @@ +export type PlayerTimeoutStats = { + user: string; + count: number; + latestTimeoutMs: number; +}; diff --git a/crons/src/types/stats/RivalryStats.ts b/crons/src/types/stats/RivalryStats.ts new file mode 100644 index 00000000..0f707c91 --- /dev/null +++ b/crons/src/types/stats/RivalryStats.ts @@ -0,0 +1,30 @@ +export type RivalryPair = { + userA: string; + userB: string; + n: number; +}; + +/** Private ops rivalry entry — includes user IDs and display names. */ +export type IdentifiedRivalryPair = { + userA: string; + nameA: string; + userB: string; + nameB: string; + n: number; +}; + +export type RivalriesFull = { + generated: string; + minGames: number; + pairs: IdentifiedRivalryPair[]; +}; + +export type AnonymizedRivalry = { + rank: number; + label: string; + n: number; + players?: { + id: string; + name: string; + }[]; +}; diff --git a/crons/src/types/stats/SeasonalityStats.ts b/crons/src/types/stats/SeasonalityStats.ts new file mode 100644 index 00000000..4f0c93e3 --- /dev/null +++ b/crons/src/types/stats/SeasonalityStats.ts @@ -0,0 +1,8 @@ +/** Move-time activity in UTC (from records-move-times). `movesByDow` / `playersByDow` index 0 = Sunday. */ +export type SeasonalityStats = { + movesByDow: number[]; + playersByDow: number[]; + movesByHour: number[]; + /** Rolling window in days used to compute these bins (typically 365). */ + windowDays: number; +}; diff --git a/crons/src/types/stats/SoloStats.ts b/crons/src/types/stats/SoloStats.ts new file mode 100644 index 00000000..f41b1c64 --- /dev/null +++ b/crons/src/types/stats/SoloStats.ts @@ -0,0 +1,45 @@ +export type SoloOutcomeType = "binary" | "graded" | "score" | "timed"; +export type ScoreDirection = "higher" | "lower"; + +/** Per meta game + variant combo (all seeds combined). */ +export type SoloMetaStats = { + game: string; + metaUid: string; + variants: string[]; + attempts: number; + uniquePlayers: number; + repeatAttemptRate: number; + outcomeTypes: Partial>; + scoreMedianAllAttempts?: number; + scoreMedianBestPerUser?: number; + scoreP90BestPerUser?: number; + passRateAllAttempts?: number; + passRateBestPerUser?: number; + gradeHistogramBestPerUser?: Record; + moveCountMedian?: number; +}; + +export type SoloSeedBoardRow = { + userid: string; + name: string; + score: number; + grade?: string; + passed?: boolean; + dateEnd: string; + attempts: number; +}; + +/** Per meta game + variant + challenge-seed leaderboard. */ +export type SoloSeedBoard = { + game: string; + metaUid: string; + variants: string[]; + challengeSeed: string; + scoreDirection: ScoreDirection; + outcomeType?: SoloOutcomeType; + attempts: number; + uniquePlayers: number; + scoreMedianAllAttempts?: number; + scoreMedianBestPerUser?: number; + rows: SoloSeedBoardRow[]; +}; diff --git a/crons/src/types/stats/StatSummary.ts b/crons/src/types/stats/StatSummary.ts new file mode 100644 index 00000000..a07286e3 --- /dev/null +++ b/crons/src/types/stats/StatSummary.ts @@ -0,0 +1,84 @@ +import { + UserRating, + UserGameRating, + GameNumber, + UserNumber, + GameNumList, + UserNumList, + TwoPlayerStats, + GeoStats, + HoursPerStats, + PlayContextStats, + MetaPieStats, + MetaPlayerCountMix, + AnonymizedRivalry, + SeasonalityStats, + GlickoByGameRow, + GlickoSiteEntry, + GlickoMeta, +} from "./index.js"; +import type { PlayerTimeoutStats } from "./PlayerTimeoutStats.js"; +import type { SoloMetaStats, SoloSeedBoard } from "./SoloStats.js"; +export type StatSummary = { + numGames: number; + numPlayers: number; + oldestRec?: string; + newestRec?: string; + timeoutRate: number; + abandonedRate: number; + playContext: PlayContextStats; + pieRates: MetaPieStats[]; + playerCountMix: MetaPlayerCountMix[]; + ratings: { + highest: UserGameRating[]; + avg: UserRating[]; + weighted: UserRating[]; + glickoByGame: GlickoByGameRow[]; + glickoSite: GlickoSiteEntry[]; + glickoMeta: GlickoMeta; + playerCountsByUid: Record; + }; + topPlayers: UserGameRating[]; + plays: { + total: GameNumber[]; + width: GameNumber[]; + }; + players: { + social: UserNumber[]; + eclectic: UserNumber[]; + allPlays: UserNumber[]; + h: UserNumber[]; + hOpp: UserNumber[]; + timeoutStats: PlayerTimeoutStats[]; + }; + histograms: { + all: number[]; + allPlayers: number[]; + meta: GameNumList[]; + players: UserNumList[]; + playerTimeouts: UserNumList[]; + firstTimers: number[]; + returningPlayers: number[]; + activeMovers: number[]; + timeouts: number[]; + abandoned: number[]; + }; + recent: GameNumber[]; + hoursPer: HoursPerStats; + metaStats: { + [k: string]: TwoPlayerStats; + } + soloMetaStats: { + [k: string]: SoloMetaStats; + }; + soloSeedBoards: SoloSeedBoard[]; + hMeta: UserNumber[]; + geoStats: GeoStats[]; + activeGeoStats: GeoStats[]; + rivalries: AnonymizedRivalry[]; + seasonality: SeasonalityStats; + pastDisplayNames: { + user: string; + names: string[]; + }[]; +}; diff --git a/crons/src/types/stats/StatSummaryTiers.ts b/crons/src/types/stats/StatSummaryTiers.ts new file mode 100644 index 00000000..9ebad3c0 --- /dev/null +++ b/crons/src/types/stats/StatSummaryTiers.ts @@ -0,0 +1,118 @@ +import type { UserGameRating, UserRating, UserNumber, UserNumList, GameNumber, GameNumList } from "./index.js"; +import type { GlickoByGameRow, GlickoSiteEntry, GlickoMeta } from "./GlickoStats.js"; +import type { PlayerTimeoutStats } from "./PlayerTimeoutStats.js"; +import type { SoloMetaStats, SoloSeedBoard } from "./SoloStats.js"; +import type { + TwoPlayerStats, + GeoStats, + HoursPerStats, + PlayContextStats, + MetaPieStats, + MetaPlayerCountMix, + AnonymizedRivalry, + SeasonalityStats, +} from "./index.js"; + +export type StatSummarySite = { + generated: string; + tier: "site"; + numGames: number; + numPlayers: number; + oldestRec?: string; + newestRec?: string; + timeoutRate: number; + abandonedRate: number; + playContext: PlayContextStats; + pieRates: MetaPieStats[]; + playerCountMix: MetaPlayerCountMix[]; + geoStats: GeoStats[]; + activeGeoStats: GeoStats[]; + seasonality: SeasonalityStats; + rivalries: AnonymizedRivalry[]; + hoursPer: HoursPerStats; + recent: GameNumber[]; + histograms: { + all: number[]; + allPlayers: number[]; + activeMovers: number[]; + returningPlayers: number[]; + firstTimers: number[]; + timeouts: number[]; + abandoned: number[]; + meta: GameNumList[]; + }; + hMeta: UserNumber[]; + metaStats: { + [k: string]: TwoPlayerStats; + }; + soloMetaStats: { + [k: string]: SoloMetaStats; + }; + soloSeedBoards: SoloSeedBoard[]; + plays: { + total: GameNumber[]; + width: GameNumber[]; + }; + topPlayers: UserGameRating[]; +}; + +export type StatSummaryPlayers = { + generated: string; + tier: "players"; + players: { + social: UserNumber[]; + eclectic: UserNumber[]; + allPlays: UserNumber[]; + h: UserNumber[]; + hOpp: UserNumber[]; + timeoutStats: PlayerTimeoutStats[]; + }; + histograms: { + players: UserNumList[]; + playerTimeouts: UserNumList[]; + }; + pastDisplayNames?: { + user: string; + names: string[]; + }[]; +}; + +export type StatSummaryRatings = { + generated: string; + tier: "ratings"; + ratings: { + highest: UserGameRating[]; + avg: UserRating[]; + weighted: UserRating[]; + glickoByGame: GlickoByGameRow[]; + glickoSite: GlickoSiteEntry[]; + glickoMeta: GlickoMeta; + playerCountsByUid: Record; + }; +}; + +export type PlayerSummarySlice = { + generated: string; + user: string; + pastDisplayNames?: string[]; + players: { + allPlays?: number; + eclectic?: number; + social?: number; + h?: number; + hOpp?: number; + timeoutCount?: number; + latestTimeoutMs?: number; + }; + histograms: { + players?: number[]; + playerTimeouts?: number[]; + }; + ratings: { + highest: UserGameRating[]; + glickoByGame?: GlickoByGameRow[]; + glickoSite?: GlickoSiteEntry; + avg?: number; + weighted?: number; + }; +}; diff --git a/crons/src/types/stats/TwoPlayerStats.ts b/crons/src/types/stats/TwoPlayerStats.ts new file mode 100644 index 00000000..9afefa73 --- /dev/null +++ b/crons/src/types/stats/TwoPlayerStats.ts @@ -0,0 +1,7 @@ +export interface TwoPlayerStats { + n: number; + lenAvg: number; + lenMedian: number; + winsFirst: number; + drawRate: number; +} diff --git a/crons/src/types/stats/UserGameRating.ts b/crons/src/types/stats/UserGameRating.ts new file mode 100644 index 00000000..004106cb --- /dev/null +++ b/crons/src/types/stats/UserGameRating.ts @@ -0,0 +1,8 @@ +import type { GlickoStats } from "./GlickoStats.js"; +import type { UserRating } from "./UserRating.js";; +export interface UserGameRating extends UserRating { + game: string; + wld: [number,number,number]; + glicko?: GlickoStats; + trueskill?: {mu: number; sigma: number}; +} diff --git a/crons/src/types/stats/UserNumList.ts b/crons/src/types/stats/UserNumList.ts new file mode 100644 index 00000000..9c0e23f1 --- /dev/null +++ b/crons/src/types/stats/UserNumList.ts @@ -0,0 +1,4 @@ +export interface UserNumList { + user: string; + value: number[]; +} diff --git a/crons/src/types/stats/UserNumber.ts b/crons/src/types/stats/UserNumber.ts new file mode 100644 index 00000000..bb1dae8e --- /dev/null +++ b/crons/src/types/stats/UserNumber.ts @@ -0,0 +1,4 @@ +export interface UserNumber { + user: string; + value: number; +} diff --git a/crons/src/types/stats/UserRating.ts b/crons/src/types/stats/UserRating.ts new file mode 100644 index 00000000..175fd7d4 --- /dev/null +++ b/crons/src/types/stats/UserRating.ts @@ -0,0 +1,4 @@ +export interface UserRating { + user: string; + rating: number; +} diff --git a/crons/src/types/stats/index.ts b/crons/src/types/stats/index.ts new file mode 100644 index 00000000..9d42607e --- /dev/null +++ b/crons/src/types/stats/index.ts @@ -0,0 +1,38 @@ +export { UserRating } from "./UserRating.js"; +export { UserGameRating } from "./UserGameRating.js"; +export type { + GlickoStats, + GlickoByGameRow, + GlickoSiteEntry, + GlickoGameCounts, + GlickoSiteCounts, + GlickoMeta, +} from "./GlickoStats.js"; +export type { StatSummary } from "./StatSummary.js"; +export type { + StatSummarySite, + StatSummaryPlayers, + StatSummaryRatings, + PlayerSummarySlice, +} from "./StatSummaryTiers.js"; +export type { PlayerTimeoutStats } from "./PlayerTimeoutStats.js"; +export { GameNumber } from "./GameNumber.js"; +export { GameNumList } from "./GameNumList.js"; +export { UserNumber } from "./UserNumber.js"; +export { UserNumList } from "./UserNumList.js"; +export { TwoPlayerStats } from "./TwoPlayerStats.js"; +export { GeoStats } from "./GeoStats.js"; +export type { HoursPerStats } from "./HoursPerStats.js"; +export type { PlayContextStats } from "./PlayContextStats.js"; +export type { MetaPieStats } from "./MetaPieStats.js"; +export type { MetaPlayerCountMix } from "./MetaPlayerCountMix.js"; +export type { RivalryPair, IdentifiedRivalryPair, RivalriesFull, AnonymizedRivalry } from "./RivalryStats.js"; +export type { SeasonalityStats } from "./SeasonalityStats.js"; +export type { + SoloMetaStats, + SoloSeedBoard, + SoloSeedBoardRow, + SoloOutcomeType, + ScoreDirection, +} from "./SoloStats.js"; + diff --git a/crons/src/utils/ReservoirSampler.ts b/crons/src/utils/ReservoirSampler.ts new file mode 100644 index 00000000..268574ad --- /dev/null +++ b/crons/src/utils/ReservoirSampler.ts @@ -0,0 +1,25 @@ +export class ReservoirSampler { + private reservoir: T[] = []; + private count = 0; + + constructor(private k: number = 1) {} + + add(item: T): void { + this.count++; + + if (this.count <= this.k) { + // Fill reservoir until it has k items + this.reservoir.push(item); + } else { + // Randomly replace an existing item + const j = Math.floor(Math.random() * this.count); + if (j < this.k) { + this.reservoir[j] = item; + } + } + } + + getSample(): T[] { + return [...this.reservoir]; + } +} diff --git a/crons/src/utils/cloudwatchMetrics.ts b/crons/src/utils/cloudwatchMetrics.ts new file mode 100644 index 00000000..7100ff32 --- /dev/null +++ b/crons/src/utils/cloudwatchMetrics.ts @@ -0,0 +1,32 @@ +import { CloudWatchClient, PutMetricDataCommand } from "@aws-sdk/client-cloudwatch"; + +const NAMESPACE = "AbstractPlay/Thumbnails"; +const client = new CloudWatchClient({}); + +/** Publish a single count metric (best-effort; logs on failure). */ +export async function putThumbnailMetric( + name: string, + value: number, + dimensions: Record = {}, +): Promise { + try { + await client.send( + new PutMetricDataCommand({ + Namespace: NAMESPACE, + MetricData: [ + { + MetricName: name, + Value: value, + Unit: "Count", + Dimensions: Object.entries(dimensions).map(([Name, Value]) => ({ + Name, + Value, + })), + }, + ], + }), + ); + } catch (err) { + console.error(`Failed to publish CloudWatch metric ${name}:`, err); + } +} diff --git a/crons/src/utils/completedGameRec.test.ts b/crons/src/utils/completedGameRec.test.ts new file mode 100644 index 00000000..649a554a --- /dev/null +++ b/crons/src/utils/completedGameRec.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it, vi } from "vitest"; +import { + completedGameRecHasState, + gameRecHasPlayableState, + resolveGameMetaGame, + skipCompletedGameWithoutState, +} from "./completedGameRec.js"; + +describe("completedGameRec", () => { + it("accepts completed games with state and players", () => { + expect(completedGameRecHasState({ + pk: "GAME", + sk: "go#1#uuid", + metaGame: "go", + state: "{}", + players: [{ id: "p1", name: "A" }], + })).toBe(true); + }); + + it("rejects completed games missing state", () => { + expect(completedGameRecHasState({ + pk: "GAME", + sk: "tablero#1#377080453", + tournament: "tablero#uuid", + })).toBe(false); + }); + + it("rejects completed games missing players even with state", () => { + expect(completedGameRecHasState({ + pk: "GAME", + sk: "go#1#uuid", + metaGame: "go", + state: "{}", + })).toBe(false); + }); + + it("resolves metaGame from sk when field is absent", () => { + expect(resolveGameMetaGame({ + sk: "volo#1#444727048", + })).toBe("volo"); + }); + + it("gameRecHasPlayableState accepts active games", () => { + expect(gameRecHasPlayableState({ + pk: "GAME", + sk: "go#0#uuid", + metaGame: "go", + state: "{}", + players: [{ id: "p1", name: "A" }, { id: "p2", name: "B" }], + })).toBe(true); + }); + + it("logs and returns true for orphan completed games", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const rec = { + pk: "GAME", + sk: "tablero#1#377080453", + tournament: "tablero#aec5b5fc-3ec4-4da9-b153-0316566660bb", + }; + expect(skipCompletedGameWithoutState(rec)).toBe(true); + expect(warn).toHaveBeenCalledWith( + "Skipping completed GAME without playable state: sk=tablero#1#377080453 keys=pk,sk,tournament", + ); + warn.mockRestore(); + }); +}); diff --git a/crons/src/utils/completedGameRec.ts b/crons/src/utils/completedGameRec.ts new file mode 100644 index 00000000..e61eaac0 --- /dev/null +++ b/crons/src/utils/completedGameRec.ts @@ -0,0 +1,58 @@ +type GameRecFields = { + pk?: string; + sk?: string; + state?: string; + metaGame?: string; + players?: unknown; +}; + +/** metaGame field, or first segment of `meta#cbit#id` sk. */ +export function resolveGameMetaGame(rec: { metaGame?: string; sk?: string }): string | undefined { + if (typeof rec.metaGame === "string" && rec.metaGame.length > 0) { + return rec.metaGame; + } + if (typeof rec.sk === "string") { + const metaGame = rec.sk.split("#")[0]; + if (metaGame.length > 0) { + return metaGame; + } + } + return undefined; +} + +function hasPlayers(rec: { players?: unknown }): boolean { + return Array.isArray(rec.players) + && rec.players.length > 0 + && rec.players.every((p) => typeof (p as { id?: string }).id === "string"); +} + +/** Active or completed GAME rows that can be passed to GameFactory / move extraction. */ +export function gameRecHasPlayableState(rec: GameRecFields): boolean { + return rec.pk === "GAME" + && typeof rec.sk === "string" + && rec.state !== undefined + && rec.state !== "" + && resolveGameMetaGame(rec) !== undefined + && hasPlayers(rec); +} + +/** Completed GAME dump rows need serialized rules state and players for GameFactory / genRecord. */ +export function completedGameRecHasState(rec: GameRecFields): boolean { + return rec.pk === "GAME" + && typeof rec.sk === "string" + && rec.sk.includes("#1#") + && gameRecHasPlayableState(rec); +} + +export function skipCompletedGameWithoutState(rec: GameRecFields & Record): boolean { + if (rec.pk !== "GAME" || typeof rec.sk !== "string" || !rec.sk.includes("#1#")) { + return false; + } + if (completedGameRecHasState(rec)) { + return false; + } + console.warn( + `Skipping completed GAME without playable state: sk=${rec.sk} keys=${Object.keys(rec).join(",")}`, + ); + return true; +} diff --git a/crons/src/utils/cooccurPmi.test.ts b/crons/src/utils/cooccurPmi.test.ts new file mode 100644 index 00000000..2852e271 --- /dev/null +++ b/crons/src/utils/cooccurPmi.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import { + buildCooccurArtifact, + computePmi, + incrementPairCounts, + pairKey, + unionCoPlaySet, +} from "./cooccurPmi.js"; + +describe("pairKey", () => { + it("orders meta games lexicographically", () => { + expect(pairKey("go", "amazons")).toBe("amazons|go"); + expect(pairKey("amazons", "go")).toBe("amazons|go"); + }); +}); + +describe("unionCoPlaySet", () => { + it("merges played and starred meta games", () => { + const set = unionCoPlaySet(["go"], ["chess", "go"]); + expect([...set].sort()).toEqual(["chess", "go"]); + }); +}); + +describe("incrementPairCounts", () => { + it("counts each unordered pair once per player", () => { + const counts = new Map(); + incrementPairCounts(new Set(["go", "amazons", "hex"]), counts); + expect(counts.get("amazons|go")).toBe(1); + expect(counts.get("go|hex")).toBe(1); + expect(counts.get("amazons|hex")).toBe(1); + expect(counts.size).toBe(3); + }); +}); + +describe("computePmi", () => { + it("returns higher PMI when co-play is disproportionate", () => { + const high = computePmi(10, 20, 20, 100); + const low = computePmi(2, 20, 20, 100); + expect(high).toBeGreaterThan(low); + }); +}); + +describe("buildCooccurArtifact", () => { + it("filters pairs below minCooccurrence", () => { + const artifact = buildCooccurArtifact( + [new Set(["go", "amazons"]), new Set(["go", "hex"])], + { minCooccurrence: 5, includeStarredBoost: false }, + ); + expect(artifact.games.go ?? []).toHaveLength(0); + }); + + it("emits PMI neighbors when pairs meet the threshold", () => { + const players = [ + ...Array.from({ length: 5 }, () => new Set(["go", "amazons"])), + ...Array.from({ length: 3 }, () => new Set(["go"])), + ...Array.from({ length: 2 }, () => new Set(["hex"])), + ]; + const artifact = buildCooccurArtifact(players, { + minCooccurrence: 5, + topK: 20, + includeStarredBoost: false, + generatedAt: "2026-08-13T00:00:00.000Z", + }); + expect(artifact.generatedAt).toBe("2026-08-13T00:00:00.000Z"); + expect(artifact.includeStarredBoost).toBe(false); + const goNeighbors = artifact.games.go ?? []; + expect(goNeighbors.some((n) => n.metaGame === "amazons" && n.count >= 5)).toBe(true); + const amazonsNeighbor = goNeighbors.find((n) => n.metaGame === "amazons"); + expect(amazonsNeighbor!.pmi).toBeGreaterThan(0); + }); + + it("starred boost adds co-play pairs without completed games", () => { + const artifact = buildCooccurArtifact( + [unionCoPlaySet(["go"], ["chess"])], + { minCooccurrence: 1, includeStarredBoost: true }, + ); + const goNeighbors = artifact.games.go ?? []; + expect(goNeighbors.some((n) => n.metaGame === "chess")).toBe(true); + }); + + it("caps neighbors at topK by PMI", () => { + const coPlay = new Set(["a", "b", "c", "d", "e", "f"]); + const players = Array.from({ length: 10 }, () => new Set(coPlay)); + const artifact = buildCooccurArtifact(players, { + minCooccurrence: 5, + topK: 2, + includeStarredBoost: false, + }); + for (const neighbors of Object.values(artifact.games)) { + expect(neighbors.length).toBeLessThanOrEqual(2); + } + }); +}); diff --git a/crons/src/utils/cooccurPmi.ts b/crons/src/utils/cooccurPmi.ts new file mode 100644 index 00000000..f947c47b --- /dev/null +++ b/crons/src/utils/cooccurPmi.ts @@ -0,0 +1,122 @@ +/** PMI co-occurrence artifact for game recommendations (front-end hybrid merge). */ + +export const DEFAULT_MIN_COOCCURRENCE = 5; +export const DEFAULT_TOP_K = 20; + +export type CooccurNeighbor = { + metaGame: string; + pmi: number; + count: number; +}; + +export type CooccurArtifact = { + generatedAt: string; + minCooccurrence: number; + /** When true, starred games were unioned into each player's co-play set. */ + includeStarredBoost: boolean; + games: Record; +}; + +/** Lexicographic pair key for unordered meta-game pair (A, B). */ +export function pairKey(a: string, b: string): string { + return a < b ? `${a}|${b}` : `${b}|${a}`; +} + +export function unionCoPlaySet(played: Iterable, starred: Iterable): Set { + const set = new Set(played); + for (const meta of starred) { + set.add(meta); + } + return set; +} + +export function incrementPairCounts(coPlaySet: Set, pairCounts: Map): void { + const games = [...coPlaySet].sort(); + for (let i = 0; i < games.length; i++) { + for (let j = i + 1; j < games.length; j++) { + const key = pairKey(games[i]!, games[j]!); + pairCounts.set(key, (pairCounts.get(key) ?? 0) + 1); + } + } +} + +export function incrementGamePlayerCounts( + coPlaySet: Set, + gamePlayerCounts: Map, +): void { + for (const meta of coPlaySet) { + gamePlayerCounts.set(meta, (gamePlayerCounts.get(meta) ?? 0) + 1); + } +} + +export function computePmi(pairCount: number, countA: number, countB: number, numPlayers: number): number { + return Math.log((pairCount * numPlayers) / (countA * countB)); +} + +export type BuildCooccurOptions = { + minCooccurrence?: number; + topK?: number; + includeStarredBoost: boolean; + generatedAt?: string; +}; + +/** + * Build the co-occurrence artifact from per-player co-play sets. + * Each set is the union of completed meta-games and (optionally) starred meta-games. + */ +export function buildCooccurArtifact( + playerCoPlaySets: Iterable>, + options: BuildCooccurOptions, +): CooccurArtifact { + const minCooccurrence = options.minCooccurrence ?? DEFAULT_MIN_COOCCURRENCE; + const topK = options.topK ?? DEFAULT_TOP_K; + const pairCounts = new Map(); + const gamePlayerCounts = new Map(); + let numPlayers = 0; + + for (const coPlaySet of playerCoPlaySets) { + if (coPlaySet.size === 0) { + continue; + } + numPlayers++; + incrementPairCounts(coPlaySet, pairCounts); + incrementGamePlayerCounts(coPlaySet, gamePlayerCounts); + } + + const neighborsByGame = new Map(); + + for (const [key, count] of pairCounts.entries()) { + if (count < minCooccurrence) { + continue; + } + const [a, b] = key.split("|") as [string, string]; + const countA = gamePlayerCounts.get(a); + const countB = gamePlayerCounts.get(b); + if (countA === undefined || countB === undefined || numPlayers === 0) { + continue; + } + const pmiAB = computePmi(count, countA, countB, numPlayers); + const pmiBA = computePmi(count, countB, countA, numPlayers); + + const listA = neighborsByGame.get(a) ?? []; + listA.push({ metaGame: b, pmi: pmiAB, count }); + neighborsByGame.set(a, listA); + + const listB = neighborsByGame.get(b) ?? []; + listB.push({ metaGame: a, pmi: pmiBA, count }); + neighborsByGame.set(b, listB); + } + + const games: Record = {}; + for (const [meta, neighbors] of neighborsByGame.entries()) { + neighbors.sort((x, y) => y.pmi - x.pmi || y.count - x.count || x.metaGame.localeCompare(y.metaGame)); + games[meta] = neighbors.slice(0, topK); + } + + return { + generatedAt: options.generatedAt ?? new Date().toISOString(), + minCooccurrence, + includeStarredBoost: options.includeStarredBoost, + games, + }; +} diff --git a/crons/src/utils/countryCodeList.ts b/crons/src/utils/countryCodeList.ts new file mode 100644 index 00000000..e83de301 --- /dev/null +++ b/crons/src/utils/countryCodeList.ts @@ -0,0 +1,520 @@ +/** + * A flag object + */ +export interface IFlagEntry { + countryName: string; + alpha2: string; + alpha3?: string; + numeric?: string; +} + +export const countryCodeList: IFlagEntry[] = [ + { countryName: "Andorra", alpha2: "AD", alpha3: "AND", numeric: "020" }, + { + countryName: "United Arab Emirates", + alpha2: "AE", + alpha3: "ARE", + numeric: "784", + }, + { countryName: "African Union", alpha2: "AFRUN" }, + { + countryName: "Antigua and Barbuda", + alpha2: "AG", + alpha3: "ATG", + numeric: "028", + }, + { countryName: "Anguilla", alpha2: "AI", alpha3: "AIA", numeric: "660" }, + { countryName: "Albania", alpha2: "AL", alpha3: "ALB", numeric: "008" }, + { countryName: "Armenia", alpha2: "AM", alpha3: "ARM", numeric: "051" }, + { countryName: "Amsterdam", alpha2: "AMS" }, + { countryName: "Angola", alpha2: "AO", alpha3: "AGO", numeric: "024" }, + { countryName: "Antarctica", alpha2: "AQ", alpha3: "ATA", numeric: "010" }, + { countryName: "Argentina", alpha2: "AR", alpha3: "ARG", numeric: "032" }, + { + countryName: "American Samoa", + alpha2: "AS", + alpha3: "ASM", + numeric: "016", + }, + { countryName: "Austria", alpha2: "AT", alpha3: "AUT", numeric: "040" }, + { countryName: "Australia", alpha2: "AU", alpha3: "AUS", numeric: "016" }, + { countryName: "Aruba", alpha2: "AW", alpha3: "ABW", numeric: "533" }, + { countryName: "Åland Islands", alpha2: "AX", alpha3: "ALA", numeric: "248" }, + { countryName: "Azerbaijan", alpha2: "AZ", alpha3: "AZE", numeric: "031" }, + { + countryName: "Bosnia and Herzegovina", + alpha2: "BA", + alpha3: "BIH", + numeric: "070", + }, + { countryName: "Barbados", alpha2: "BB", alpha3: "BRB", numeric: "052" }, + { countryName: "Bangladesh", alpha2: "BD", alpha3: "BGD", numeric: "050" }, + { countryName: "Belgium", alpha2: "BE", alpha3: "BEL", numeric: "056" }, + { countryName: "Burkina Faso", alpha2: "BF", alpha3: "BFA", numeric: "854" }, + { countryName: "Bulgaria", alpha2: "BG", alpha3: "BGR", numeric: "100" }, + { countryName: "Bahrain", alpha2: "BH", alpha3: "BHR", numeric: "048" }, + { countryName: "Burundi", alpha2: "BI", alpha3: "BDI", numeric: "108" }, + { countryName: "Benin", alpha2: "BJ", alpha3: "BEN", numeric: "204" }, + { + countryName: "Saint Barthélemy", + alpha2: "BL", + alpha3: "BLM", + numeric: "652", + }, + { countryName: "Bermuda", alpha2: "BM", alpha3: "BMU", numeric: "060" }, + { countryName: "Brunei", alpha2: "BN", alpha3: "BRN", numeric: "096" }, + { countryName: "Bolivia", alpha2: "BO", alpha3: "BOL", numeric: "068" }, + { countryName: "Bonaire", alpha2: "BQ-BO" }, + { countryName: "Saba", alpha2: "BQ-SA" }, + { countryName: "Sint Eustatius", alpha2: "BQ-SE" }, + { countryName: "Brazil", alpha2: "BR", alpha3: "BRA", numeric: "076" }, + { countryName: "Bahamas", alpha2: "BS", alpha3: "BHS", numeric: "044" }, + { countryName: "Bhutan", alpha2: "BT", alpha3: "BTN", numeric: "064" }, + { countryName: "Botswana", alpha2: "BW", alpha3: "BWA", numeric: "072" }, + { countryName: "Belarus", alpha2: "BY", alpha3: "BLR", numeric: "112" }, + { countryName: "Belize", alpha2: "BZ", alpha3: "BLZ", numeric: "084" }, + { countryName: "Canada", alpha2: "CA", alpha3: "CAN", numeric: "124" }, + { countryName: "Cocos Islands", alpha2: "CC", alpha3: "CCK", numeric: "166" }, + { + countryName: "Democratis Republic of Congo", + alpha2: "CD", + alpha3: "COD", + numeric: "180", + }, + { + countryName: "Central African Republic", + alpha2: "CF", + alpha3: "CAF", + numeric: "140", + }, + { countryName: "Congo", alpha2: "CG", alpha3: "COG", numeric: "178" }, + { countryName: "Switzerland", alpha2: "CH", alpha3: "CHE", numeric: "756" }, + { countryName: "Côte d'Ivoire", alpha2: "CI", alpha3: "CIV", numeric: "384" }, + { countryName: "Cook Island", alpha2: "CK", alpha3: "COK", numeric: "184" }, + { countryName: "Chile", alpha2: "CL", alpha3: "CHL", numeric: "152" }, + { countryName: "Cameroon", alpha2: "CM", alpha3: "CMR", numeric: "120" }, + { countryName: "China", alpha2: "CN", alpha3: "CHN", numeric: "156" }, + { countryName: "Colombia", alpha2: "CO", alpha3: "COL", numeric: "170" }, + { countryName: "Costa Rica", alpha2: "CR", alpha3: "CRI", numeric: "188" }, + { countryName: "Cuba", alpha2: "CU", alpha3: "CUB", numeric: "192" }, + { countryName: "Cape Verde", alpha2: "CV", alpha3: "CPV", numeric: "132" }, + { countryName: "Curaçao", alpha2: "CW", alpha3: "CUW", numeric: "531" }, + { + countryName: "Christmas Island", + alpha2: "CX", + alpha3: "CXR", + numeric: "162", + }, + { countryName: "Cyprus", alpha2: "CY", alpha3: "CYP", numeric: "196" }, + { + countryName: "Czech Republic", + alpha2: "CZ", + alpha3: "CZE", + numeric: "203", + }, + { countryName: "Germany", alpha2: "DE", alpha3: "DEU", numeric: "276" }, + { countryName: "Djibouti", alpha2: "DJ", alpha3: "DJI", numeric: "262" }, + { countryName: "Denmark", alpha2: "DK", alpha3: "DNK", numeric: "208" }, + { countryName: "Dominica", alpha2: "DM", alpha3: "DMA", numeric: "212" }, + { + countryName: "Dominican Republic", + alpha2: "DO", + alpha3: "DOM", + numeric: "214", + }, + { countryName: "Ecuador", alpha2: "EC", alpha3: "ECU", numeric: "218" }, + { countryName: "Estonia", alpha2: "EE", alpha3: "EST", numeric: "233" }, + { countryName: "Egypt", alpha2: "EG", alpha3: "EGY", numeric: "818" }, + { + countryName: "Western Sahara", + alpha2: "EH", + alpha3: "ESH", + numeric: "732", + }, + { countryName: "Eritrea", alpha2: "ER", alpha3: "ERI", numeric: "232" }, + { countryName: "Spain", alpha2: "ES", alpha3: "ESP", numeric: "724" }, + { countryName: "Ethiopia", alpha2: "ET", alpha3: "ETH", numeric: "231" }, + { countryName: "European Union", alpha2: "EU" }, + { countryName: "Finland", alpha2: "FI", alpha3: "FIN", numeric: "246" }, + { countryName: "Fiji", alpha2: "FJ", alpha3: "FJI", numeric: "242" }, + { + countryName: "Falkland Islands", + alpha2: "FK", + alpha3: "FLK", + numeric: "238", + }, + { + countryName: "Micronesia (Federated States of)", + alpha2: "FM", + alpha3: "FSM", + numeric: "583", + }, + { countryName: "Faroe Island", alpha2: "FO", alpha3: "FRO", numeric: "234" }, + { countryName: "France", alpha2: "FR", alpha3: "FRA", numeric: "250" }, + { countryName: "Gabon", alpha2: "GA", alpha3: "GAB", numeric: "266" }, + { countryName: "England", alpha2: "GB-ENG" }, + { countryName: "Scotland", alpha2: "GB-SCT" }, + { + countryName: "United Kingdom", + alpha2: "GB-UKM", + alpha3: "GBR", + numeric: "836", + }, + { countryName: "Wales", alpha2: "GB-WLS" }, + { countryName: "Northern Ireland", alpha2: "GB-NIR" }, + { countryName: "Grenada", alpha2: "GD", alpha3: "GRD", numeric: "308" }, + { countryName: "Georgia", alpha2: "GE", alpha3: "GEO", numeric: "268" }, + { countryName: "French Guiana", alpha2: "GF", alpha3: "GUF", numeric: "254" }, + { countryName: "Guernsey", alpha2: "GG", alpha3: "GGY", numeric: "831" }, + { countryName: "Ghana", alpha2: "GH", alpha3: "GHA", numeric: "288" }, + { countryName: "Gibraltar", alpha2: "GI", alpha3: "GIB", numeric: "292" }, + { countryName: "Greenland", alpha2: "GL", alpha3: "GRL", numeric: "304" }, + { countryName: "Gambia", alpha2: "GM", alpha3: "GMB", numeric: "270" }, + { countryName: "Guinea", alpha2: "GN", alpha3: "GIN", numeric: "324" }, + { countryName: "Guadeloupe", alpha2: "GP", alpha3: "GLP", numeric: "312" }, + { + countryName: "Equatorial Guinea", + alpha2: "GQ", + alpha3: "GNQ", + numeric: "226", + }, + { countryName: "Greece", alpha2: "GR", alpha3: "GRC", numeric: "300" }, + { + countryName: "South Gerogia and the South Sandwich Islands", + alpha2: "GS", + alpha3: "SGS", + numeric: "239", + }, + { countryName: "Guatemala", alpha2: "GT", alpha3: "GTM", numeric: "320" }, + { countryName: "Guam", alpha2: "GU", alpha3: "GUM", numeric: "316" }, + { countryName: "Guinea-Bissau", alpha2: "GW", alpha3: "GNB", numeric: "624" }, + { countryName: "Guyana", alpha2: "GY", alpha3: "GUY", numeric: "328" }, + { countryName: "Hong Kong", alpha2: "HK", alpha3: "HKG", numeric: "344" }, + { + countryName: "Heard Island and McDonald Islands", + alpha2: "HM", + alpha3: "HMD", + numeric: "334", + }, + { countryName: "Honduras", alpha2: "HN", alpha3: "HND", numeric: "340" }, + { countryName: "Croatia", alpha2: "HR", alpha3: "HRV", numeric: "191" }, + { countryName: "Haiti", alpha2: "HT", alpha3: "HTI", numeric: "332" }, + { countryName: "Hungary", alpha2: "HU", alpha3: "HUN", numeric: "348" }, + { countryName: "Indonesia", alpha2: "ID", alpha3: "IDN", numeric: "360" }, + { countryName: "Ireland", alpha2: "IE", alpha3: "IRL", numeric: "372" }, + { countryName: "Israel", alpha2: "IL", alpha3: "ISR", numeric: "376" }, + { countryName: "Isle of Man", alpha2: "IM", alpha3: "IMN", numeric: "833" }, + { countryName: "India", alpha2: "IN", alpha3: "IND", numeric: "356" }, + { + countryName: "British Indian Ocean Territory", + alpha2: "IO", + alpha3: "IOT", + numeric: "086", + }, + { countryName: "Iraq", alpha2: "IQ", alpha3: "IRQ", numeric: "368" }, + { countryName: "Iran", alpha2: "IR", alpha3: "IRN", numeric: "364" }, + { countryName: "Iceland", alpha2: "IS", alpha3: "ISL", numeric: "352" }, + { countryName: "Italy", alpha2: "IT", alpha3: "ITA", numeric: "380" }, + { countryName: "Jersey", alpha2: "JE", alpha3: "JEY", numeric: "832" }, + { countryName: "Jamaica", alpha2: "JM", alpha3: "JAM", numeric: "388" }, + { countryName: "Jordan", alpha2: "JO", alpha3: "JOR", numeric: "400" }, + { countryName: "Japan", alpha2: "JP", alpha3: "JPN", numeric: "392" }, + { countryName: "Kenya", alpha2: "KE", alpha3: "KEN", numeric: "404" }, + { countryName: "Kyrgyzstan", alpha2: "KG", alpha3: "KGZ", numeric: "417" }, + { countryName: "Cambodia", alpha2: "KH", alpha3: "KHM", numeric: "116" }, + { countryName: "Kiribati", alpha2: "KI", alpha3: "KIR", numeric: "296" }, + { countryName: "Comoros", alpha2: "KM", alpha3: "COM", numeric: "174" }, + { + countryName: "Saint Kitts and Nevis", + alpha2: "KN-SK", + alpha3: "KNA", + numeric: "659", + }, + { + countryName: "Korea (the Democratic People's Republic of)", + alpha2: "KP", + alpha3: "PRK", + numeric: "408", + }, + { + countryName: "Korea (the Republic of)", + alpha2: "KR", + alpha3: "KOR", + numeric: "410", + }, + { countryName: "Kuwait", alpha2: "KW", alpha3: "KWT", numeric: "414" }, + { + countryName: "Cayman Islands", + alpha2: "KY", + alpha3: "CYM", + numeric: "136", + }, + { countryName: "Kazakhstan", alpha2: "KZ", alpha3: "KAZ", numeric: "398" }, + { + countryName: "Lao People's Democratic Republic (the)", + alpha2: "LA", + alpha3: "LAO", + numeric: "418", + }, + { countryName: "Lebanon", alpha2: "LB", alpha3: "LBN", numeric: "422" }, + { countryName: "Saint Lucia", alpha2: "LC", alpha3: "LCA", numeric: "662" }, + { countryName: "Liechtenstein", alpha2: "LI", alpha3: "LIE", numeric: "438" }, + { countryName: "Sri Lanka", alpha2: "LK", alpha3: "LKA", numeric: "144" }, + { countryName: "Liberia", alpha2: "LR", alpha3: "LBR", numeric: "430" }, + { countryName: "Lesotho", alpha2: "LS", alpha3: "LSO", numeric: "426" }, + { countryName: "Lithuania", alpha2: "LT", alpha3: "LTU", numeric: "440" }, + { countryName: "Luxembourg", alpha2: "LU", alpha3: "LUX", numeric: "442" }, + { countryName: "Latvia", alpha2: "LV", alpha3: "LVA", numeric: "428" }, + { countryName: "Libya", alpha2: "LY", alpha3: "LBY", numeric: "434" }, + { countryName: "Morocco", alpha2: "MA", alpha3: "MAR", numeric: "504" }, + { countryName: "Monaco", alpha2: "MC", alpha3: "MCO ", numeric: "492" }, + { + countryName: "Moldova (the Republic of)", + alpha2: "MD", + alpha3: "MDA", + numeric: "498", + }, + { countryName: "Montenegro", alpha2: "ME", alpha3: "MNE", numeric: "499" }, + { countryName: "Saint Martin", alpha2: "MF", alpha3: "MAF", numeric: "663" }, + { countryName: "Madagascar", alpha2: "MG", alpha3: "MDG", numeric: "450" }, + { + countryName: "Marshall Islands (the)", + alpha2: "MH", + alpha3: "MHL", + numeric: "584", + }, + { + countryName: "North Macedonia", + alpha2: "MK", + alpha3: "MKD", + numeric: "807", + }, + { countryName: "Mali", alpha2: "ML", alpha3: "MLI", numeric: "466" }, + { countryName: "Myanmar", alpha2: "MM", alpha3: "MMR", numeric: "104" }, + { countryName: "Mongolia", alpha2: "MN", alpha3: "MNG", numeric: "496" }, + { countryName: "Macao", alpha2: "MO", alpha3: "MAC", numeric: "446" }, + { + countryName: "Northern Mariana Islands (the)", + alpha2: "MP", + alpha3: "MNP", + numeric: "580", + }, + { countryName: "Martinique", alpha2: "MQ", alpha3: "MTQ", numeric: "474" }, + { countryName: "Mauritania", alpha2: "MR", alpha3: "MRT", numeric: "478" }, + { countryName: "Montserrat", alpha2: "MS", alpha3: "MSR", numeric: "500" }, + { countryName: "Malta", alpha2: "MT", alpha3: "MLT", numeric: "470" }, + { countryName: "Mauritius", alpha2: "MU", alpha3: "MUS", numeric: "480" }, + { countryName: "Maldives", alpha2: "MV", alpha3: "MDV", numeric: "462" }, + { countryName: "Malawi", alpha2: "MW", alpha3: "MWI", numeric: "454" }, + { countryName: "Mexico", alpha2: "MX", alpha3: "MEX", numeric: "484" }, + { countryName: "Malaysia", alpha2: "MY", alpha3: "MYS", numeric: "458" }, + { countryName: "Mozambique", alpha2: "MZ", alpha3: "MOZ", numeric: "508" }, + { countryName: "Namibia", alpha2: "NA", alpha3: "NAM", numeric: "516" }, + { countryName: "New Caledonia", alpha2: "NC", alpha3: "NCL", numeric: "540" }, + { countryName: "Niger", alpha2: "NE", alpha3: "NER", numeric: "562" }, + { + countryName: "Norfolk Island", + alpha2: "NF", + alpha3: "NFK", + numeric: "574", + }, + { countryName: "Nigeria", alpha2: "NG", alpha3: "NGA", numeric: "566" }, + { countryName: "Nicaragua", alpha2: "NI", alpha3: "NIC", numeric: "558" }, + { countryName: "Netherlands", alpha2: "NL", alpha3: "NLD", numeric: "528" }, + { countryName: "Norway", alpha2: "NO", alpha3: "NOR", numeric: "578" }, + { countryName: "Nepal", alpha2: "NP", alpha3: "NPL", numeric: "524" }, + { countryName: "Nauru", alpha2: "NR", alpha3: "NRU", numeric: "520" }, + { countryName: "Niue", alpha2: "NU", alpha3: "NIU", numeric: "570" }, + { countryName: "New Zealand", alpha2: "NZ", alpha3: "NZL", numeric: "554" }, + { countryName: "Oman", alpha2: "OM", alpha3: "OMN", numeric: "512" }, + { countryName: "Panama", alpha2: "PA", alpha3: "PAN", numeric: "591" }, + { countryName: "Peru", alpha2: "PE", alpha3: "PER", numeric: "604" }, + { + countryName: "French Polyesia", + alpha2: "PF", + alpha3: "PYF", + numeric: "258", + }, + { + countryName: "Papua New Guinea", + alpha2: "PG", + alpha3: "PNG", + numeric: "598", + }, + { + countryName: "Phillippines (the)", + alpha2: "PH", + alpha3: "PHL", + numeric: "608", + }, + { countryName: "Pakistan", alpha2: "PK", alpha3: "PAK", numeric: "586" }, + { countryName: "Poland", alpha2: "PL", alpha3: "POL", numeric: "616" }, + { + countryName: "Saint Pierre and Miquelon", + alpha2: "PM", + alpha3: "SPM", + numeric: "666", + }, + { countryName: "Pitcairn", alpha2: "PN", alpha3: "PCN", numeric: "612" }, + { countryName: "Puerto Rico", alpha2: "PR", alpha3: "PRI", numeric: "630" }, + { + countryName: "Palestine, State of", + alpha2: "PS", + alpha3: "PSE", + numeric: "275", + }, + { countryName: "Portugal", alpha2: "PT", alpha3: "PRT", numeric: "620" }, + { countryName: "Palau", alpha2: "PW", alpha3: "PLW", numeric: "585" }, + { countryName: "Paraguay", alpha2: "PY", alpha3: "PRY", numeric: "600" }, + { countryName: "Qatar", alpha2: "QA", alpha3: "QAT", numeric: "634" }, + { countryName: "Rainbow", alpha2: "RAINBOW" }, + { countryName: "Réunion", alpha2: "RE", alpha3: "REU", numeric: "638" }, + { countryName: "Romania", alpha2: "RO", alpha3: "ROU", numeric: "642" }, + { countryName: "Serbia", alpha2: "RS", alpha3: "SRB", numeric: "688" }, + { + countryName: "Russian Federation (the)", + alpha2: "RU", + alpha3: "RUS", + numeric: "643", + }, + { countryName: "Rwanda", alpha2: "RW", alpha3: "RWA", numeric: "646" }, + { countryName: "Saudi Arabia", alpha2: "SA", alpha3: "SAU", numeric: "682" }, + { + countryName: "Solomon Islands", + alpha2: "SB", + alpha3: "SLB", + numeric: "090", + }, + { countryName: "Seychelles", alpha2: "SC", alpha3: "SYC", numeric: "690" }, + { countryName: "Sudan (the)", alpha2: "SD", alpha3: "SDN", numeric: "729" }, + { countryName: "Sweden", alpha2: "SE", alpha3: "SWE", numeric: "752" }, + { countryName: "Singapore", alpha2: "SG", alpha3: "SGP", numeric: "702" }, + { + countryName: "Saint Helena, Ascension Island, Traistan da Cunha", + alpha2: "SH", + alpha3: "SHN", + numeric: "654", + }, + { countryName: "Slovenia", alpha2: "SI", alpha3: "SVN", numeric: "705" }, + { + countryName: "Svalbard, Jan Mayen", + alpha2: "SJ", + alpha3: "SJM", + numeric: "744", + }, + { countryName: "Slovakia", alpha2: "SK", alpha3: "SVK", numeric: "703" }, + { countryName: "Sierra Leone", alpha2: "SL", alpha3: "SLE", numeric: "694" }, + { countryName: "San Marino", alpha2: "SM", alpha3: "SMR", numeric: "674" }, + { countryName: "Senegal", alpha2: "SN", alpha3: "SEN", numeric: "686" }, + { countryName: "Somalia", alpha2: "SO", alpha3: "SOM", numeric: "706" }, + { countryName: "Suriname", alpha2: "SR", alpha3: "SUR", numeric: "740" }, + { countryName: "South Sudan", alpha2: "SS", alpha3: "SSD", numeric: "728" }, + { + countryName: "Sao Tome and Principe", + alpha2: "ST", + alpha3: "STP", + numeric: "678", + }, + { countryName: "El Salvador", alpha2: "SV", alpha3: "SLV", numeric: "222" }, + { countryName: "Sint Maarten", alpha2: "SX", alpha3: "SXM", numeric: "534" }, + { + countryName: "Syrian Arab Republic (the)", + alpha2: "SY", + alpha3: "SYR", + numeric: "760", + }, + { countryName: "Eswatini", alpha2: "SZ", alpha3: "SWZ", numeric: "748" }, + { + countryName: "Turks and Caicos Islands (the)", + alpha2: "TC", + alpha3: "TCA", + numeric: "796", + }, + { countryName: "Chad", alpha2: "TD", alpha3: "TCD", numeric: "148" }, + { + countryName: "French Southern Territories", + alpha2: "TF", + alpha3: "ATF", + numeric: "260", + }, + { countryName: "Togo", alpha2: "TG", alpha3: "TGO", numeric: "768" }, + { countryName: "Thailand", alpha2: "TH", alpha3: "THA", numeric: "764" }, + { countryName: "Tajikistan", alpha2: "TJ", alpha3: "TJK", numeric: "762" }, + { countryName: "Tokelau", alpha2: "TK", alpha3: "TKL", numeric: "772" }, + { countryName: "Timor-Leste", alpha2: "TL", alpha3: "TLS", numeric: "626" }, + { countryName: "Turkmenistan", alpha2: "TM", alpha3: "TKM", numeric: "795" }, + { countryName: "Tunisia", alpha2: "TN", alpha3: "TUN", numeric: "788" }, + { countryName: "Tonga", alpha2: "TO", alpha3: "TON", numeric: "776" }, + { countryName: "Turkey", alpha2: "TR", alpha3: "TUR", numeric: "792" }, + { + countryName: "Trinidad and Tobago", + alpha2: "TT", + alpha3: "TTO", + numeric: "780", + }, + { countryName: "Tuvalu", alpha2: "TV", alpha3: "TUV", numeric: "798" }, + { countryName: "Taiwan", alpha2: "TW", alpha3: "TWN", numeric: "158" }, + { + countryName: "Tanzania, the United Republic of", + alpha2: "TZ", + alpha3: "TZA", + numeric: "834", + }, + { countryName: "Ukraine", alpha2: "UA", alpha3: "UKR", numeric: "804" }, + { countryName: "Uganda", alpha2: "UG", alpha3: "UGA", numeric: "800" }, + { + countryName: "United States Minor Outlying Islands (the)", + alpha2: "UM", + alpha3: "UMI", + numeric: "581", + }, + { countryName: "Union of South American Nations", alpha2: "UNASUR" }, + { + countryName: "United States of America", + alpha2: "US", + alpha3: "USA", + numeric: "840", + }, + { countryName: "Uruguay", alpha2: "UY", alpha3: "URY", numeric: "858" }, + { countryName: "Uzbekistan", alpha2: "UZ", alpha3: "UZB", numeric: "860" }, + { countryName: "Holy See", alpha2: "VA", alpha3: "VAT", numeric: "336" }, + { + countryName: "Saint Vincent and the Grenadines", + alpha2: "VC", + alpha3: "VCT", + numeric: "670", + }, + { + countryName: "Venezuela (Bolivarian Republic of)", + alpha2: "VE", + alpha3: "VEN", + numeric: "862", + }, + { + countryName: "Virgin Islands (British)", + alpha2: "VG", + alpha3: "VGB", + numeric: "092", + }, + { + countryName: "Virgin Islands (U.S.)", + alpha2: "VI", + alpha3: "VIR", + numeric: "850", + }, + { countryName: "Vietnam", alpha2: "VN", alpha3: "VNM", numeric: "704" }, + { countryName: "Vanuatu", alpha2: "VU", alpha3: "VUT", numeric: "548" }, + { + countryName: "Wallis and Futuna", + alpha2: "WF", + alpha3: "WLF", + numeric: "876", + }, + { countryName: "Samoa", alpha2: "WS", alpha3: "WSM", numeric: "882" }, + { countryName: "Yemen", alpha2: "YE", alpha3: "YEM", numeric: "887" }, + { countryName: "South Africa", alpha2: "ZA", alpha3: "ZAF", numeric: "710" }, + { countryName: "Zambia", alpha2: "ZM", alpha3: "ZMB", numeric: "894" }, + { countryName: "Zimbabwe", alpha2: "ZW", alpha3: "ZWE", numeric: "716" }, + { countryName: "Afghanistan", alpha2: "AF", alpha3: "AFG", numeric: "004" }, + { countryName: "Bouvet Island", alpha2: "BV", alpha3: "BVT", numeric: "074" }, +]; diff --git a/crons/src/utils/dashboardCruftCleanup.test.ts b/crons/src/utils/dashboardCruftCleanup.test.ts new file mode 100644 index 00000000..af7a0d81 --- /dev/null +++ b/crons/src/utils/dashboardCruftCleanup.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { cleanupUserDashboardCruft } from './dashboardCruftCleanup.js'; + +type Store = Map>; + +function storeKey(pk: string, sk: string): string { + return `${pk}:${sk}`; +} + +function makeClient(store: Store) { + return { + send: async (command: { constructor: { name: string }; input: Record }) => { + const input = command.input; + if (command.constructor.name === 'QueryCommand') { + const pk = (input.ExpressionAttributeValues as Record)[':pk']; + const items = [...store.values()].filter(item => item.pk === pk); + return { Items: items.map(item => ({ ...item })) }; + } + if (command.constructor.name === 'DeleteCommand') { + const key = input.Key as { pk: string; sk: string }; + store.delete(storeKey(key.pk, key.sk)); + return {}; + } + throw new Error(`Unhandled ${command.constructor.name}`); + }, + }; +} + +describe('cleanupUserDashboardCruft', () => { + it('removes stale recent rows and orphan overlays', async () => { + const userId = 'user-1'; + const store: Store = new Map([ + [storeKey(`RECENTCOMPLETED#${userId}`, 'stale'), { + pk: `RECENTCOMPLETED#${userId}`, + sk: 'stale', + toMove: '', + }], + [storeKey(`USERGAME#${userId}`, 'stale'), { + pk: `USERGAME#${userId}`, + sk: 'stale', + seen: 1, + }], + [storeKey(`USERGAME#${userId}`, 'orphan'), { + pk: `USERGAME#${userId}`, + sk: 'orphan', + seen: 2, + }], + ]); + + const stats = await cleanupUserDashboardCruft( + makeClient(store) as never, + 'abstract-play-test', + userId, + Date.parse('2026-08-24T12:00:00.000Z'), + ); + + expect(stats.recentCompletedDeleted).toBe(1); + expect(stats.userGameDeleted).toBe(2); + expect(store.has(storeKey(`RECENTCOMPLETED#${userId}`, 'stale'))).toBe(false); + expect(store.has(storeKey(`USERGAME#${userId}`, 'orphan'))).toBe(false); + }); +}); diff --git a/crons/src/utils/dashboardCruftCleanup.ts b/crons/src/utils/dashboardCruftCleanup.ts new file mode 100644 index 00000000..0cc99b9c --- /dev/null +++ b/crons/src/utils/dashboardCruftCleanup.ts @@ -0,0 +1,167 @@ +/** + * Index-only dashboard cruft cleanup. + * Keep in sync with node-backend lib/dashboardCruftCleanup.ts + */ +import { + DeleteCommand, + DynamoDBDocumentClient, + QueryCommand, +} from '@aws-sdk/lib-dynamodb'; +import { removeDashboardGameMembership } from './dashboardEviction.js'; + +const COMPLETED_DASHBOARD_RETENTION_MS = 7 * 24 * 3600000; + +export type DashboardCruftCleanupStats = { + recentCompletedDeleted: number; + userGameDeleted: number; +}; + +type OverlayFields = { + seen?: number; + lastChat?: number; +}; + +type RecentRow = { + sk: string; + metaGame: string; + players: { id: string; name: string }[]; + clockHard: boolean; + noExplore?: boolean; + toMove?: string | boolean[]; + lastMoveTime: number; + variants?: string[]; + gameStarted?: number; + gameEnded?: number; + winner?: number[]; + numMoves?: number; + commented?: number; +}; + +function shouldBeOnCompletedDashboard( + game: { toMove?: string | boolean[] | null; seen?: number; lastChat?: number }, + now: number, +): boolean { + if (game.toMove !== '' && game.toMove !== null && game.toMove !== undefined) { + return false; + } + if (game.seen === undefined) { + return true; + } + if ((game.lastChat || 0) > game.seen) { + return true; + } + return now - game.seen <= COMPLETED_DASHBOARD_RETENTION_MS; +} + +function applyOverlayFields( + game: T, + overlay: OverlayFields | undefined, +): T { + const result = { ...game }; + if (overlay?.seen !== undefined) { + result.seen = overlay.seen; + } else { + delete result.seen; + } + if (overlay?.lastChat !== undefined) { + result.lastChat = overlay.lastChat; + } else { + delete result.lastChat; + } + return result; +} + +async function queryPartition( + client: DynamoDBDocumentClient, + tableName: string, + pk: string, +): Promise[]> { + const items: Record[] = []; + let lastEvaluatedKey: Record | undefined; + + do { + const page = await client.send(new QueryCommand({ + TableName: tableName, + KeyConditionExpression: '#pk = :pk', + ExpressionAttributeNames: { '#pk': 'pk' }, + ExpressionAttributeValues: { ':pk': pk }, + ExclusiveStartKey: lastEvaluatedKey, + })); + if (page.Items) { + items.push(...page.Items); + } + lastEvaluatedKey = page.LastEvaluatedKey; + } while (lastEvaluatedKey); + + return items; +} + +async function deleteRow( + client: DynamoDBDocumentClient, + tableName: string, + pk: string, + sk: string, +): Promise { + await client.send(new DeleteCommand({ + TableName: tableName, + Key: { pk, sk }, + })); +} + +export async function cleanupUserDashboardCruft( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, + now = Date.now(), +): Promise { + const [currentRows, recentRows, overlayRows] = await Promise.all([ + queryPartition(client, tableName, `CURRENTGAMES#${userId}`), + queryPartition(client, tableName, `RECENTCOMPLETED#${userId}`), + queryPartition(client, tableName, `USERGAME#${userId}`), + ]); + + const currentIds = new Set(currentRows.map(row => String(row.sk))); + const overlays = new Map(); + for (const row of overlayRows) { + overlays.set(String(row.sk), { + seen: typeof row.seen === 'number' ? row.seen : undefined, + lastChat: typeof row.lastChat === 'number' ? row.lastChat : undefined, + }); + } + + let recentCompletedDeleted = 0; + let userGameDeleted = 0; + const eligibleRecentIds = new Set(); + + for (const row of recentRows as RecentRow[]) { + const merged = applyOverlayFields({ + toMove: row.toMove ?? '', + seen: overlays.get(row.sk)?.seen, + lastChat: overlays.get(row.sk)?.lastChat, + }, overlays.get(row.sk)); + if (shouldBeOnCompletedDashboard(merged, now)) { + eligibleRecentIds.add(row.sk); + continue; + } + const evicted = await removeDashboardGameMembership( + client, + tableName, + userId, + [row.sk], + ); + overlays.delete(row.sk); + recentCompletedDeleted += evicted.recentCompletedDeleted; + userGameDeleted += evicted.userGameDeleted; + } + + const dashboardIds = new Set([...currentIds, ...eligibleRecentIds]); + for (const gameId of overlays.keys()) { + if (dashboardIds.has(gameId)) { + continue; + } + await deleteRow(client, tableName, `USERGAME#${userId}`, gameId); + userGameDeleted += 1; + } + + return { recentCompletedDeleted, userGameDeleted }; +} diff --git a/crons/src/utils/dashboardEviction.ts b/crons/src/utils/dashboardEviction.ts new file mode 100644 index 00000000..4a1ec253 --- /dev/null +++ b/crons/src/utils/dashboardEviction.ts @@ -0,0 +1,46 @@ +/** + * Idempotent RECENTCOMPLETED# + USERGAME# eviction. + * Keep in sync with node-backend lib/dashboardEviction.ts + */ +import { + DeleteCommand, + DynamoDBDocumentClient, +} from '@aws-sdk/lib-dynamodb'; + +export type DashboardEvictionStats = { + recentCompletedDeleted: number; + userGameDeleted: number; +}; + +async function deleteRow( + client: DynamoDBDocumentClient, + tableName: string, + pk: string, + sk: string, +): Promise { + await client.send(new DeleteCommand({ + TableName: tableName, + Key: { pk, sk }, + })); +} + +export async function removeDashboardGameMembership( + client: DynamoDBDocumentClient, + tableName: string, + userId: string, + gameIds: string[], +): Promise { + if (gameIds.length === 0) { + return { recentCompletedDeleted: 0, userGameDeleted: 0 }; + } + + await Promise.all(gameIds.map(gameId => Promise.all([ + deleteRow(client, tableName, `RECENTCOMPLETED#${userId}`, gameId), + deleteRow(client, tableName, `USERGAME#${userId}`, gameId), + ]))); + + return { + recentCompletedDeleted: gameIds.length, + userGameDeleted: gameIds.length, + }; +} diff --git a/crons/src/utils/dumpExport.ts b/crons/src/utils/dumpExport.ts new file mode 100644 index 00000000..c73a67da --- /dev/null +++ b/crons/src/utils/dumpExport.ts @@ -0,0 +1,123 @@ +'use strict'; + +/** + * Shared helpers for reading the daily DynamoDB ION export in S3. + * Keep in sync with node-backend abandoned-account cleanup plan when changing dump selection. + */ +import { + GetObjectCommand, + ListObjectsV2Command, + type S3Client, + type _Object, +} from '@aws-sdk/client-s3'; +import { gunzipSync, strFromU8 } from 'fflate'; +import { load as loadIon } from 'ion-js'; +import type { BasicRec } from '../types/index.js'; + +export const DUMP_BUCKET = 'abstractplay-db-dump'; + +export async function listDumpBucketObjects(s3: S3Client): Promise<_Object[]> { + const command = new ListObjectsV2Command({ Bucket: DUMP_BUCKET }); + const allContents: _Object[] = []; + let isTruncated = true; + + while (isTruncated) { + const { Contents, IsTruncated, NextContinuationToken } = await s3.send(command); + if (Contents === undefined) { + throw new Error('Could not list dump bucket contents'); + } + allContents.push(...Contents); + isTruncated = IsTruncated ?? false; + command.input.ContinuationToken = NextContinuationToken; + } + + return allContents; +} + +export function findLatestDumpUid(allContents: _Object[]): string { + const manifests = allContents.filter(c => c.Key?.includes('manifest-summary.json')); + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); + const latest = manifests[0]; + if (latest?.Key === undefined) { + throw new Error('No manifest-summary.json found in dump bucket'); + } + const match = latest.Key.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); + if (match === null) { + throw new Error(`Could not extract uid from "${latest.Key}"`); + } + return match[1]!; +} + +export function dumpDataFilesForUid(allContents: _Object[], uid: string): _Object[] { + return allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith('.ion.gz')); +} + +export async function forEachIonItem( + s3: S3Client, + file: _Object, + onItem: (item: Record) => void, +): Promise { + if (file.Key === undefined) { + return; + } + const response = await s3.send(new GetObjectCommand({ + Bucket: DUMP_BUCKET, + Key: file.Key, + })); + const bytes = await response.Body?.transformToByteArray(); + if (bytes === undefined) { + throw new Error(`Could not load bytes from file ${file.Key}`); + } + + const ion = gunzipSync(bytes); + let sofar = ''; + let ptr = 0; + const chunk = 1_000_000; + while (ptr < ion.length) { + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); + while (sofar.includes('}}\n')) { + const idx = sofar.indexOf('}}\n'); + const line = sofar.substring(0, idx + 2); + sofar = sofar.substring(idx + 3); + try { + const outerRec = loadIon(line); + if (outerRec === null) { + continue; + } + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; + onItem(json.Item as Record); + } catch { + // skip malformed lines + } + } + ptr += chunk; + } +} + +export async function collectUserCandidatesFromDump( + s3: S3Client, + allContents: _Object[], + uid: string, + inactiveBeforeMs: number, +): Promise { + const candidates = new Set(); + const dataFiles = dumpDataFilesForUid(allContents, uid); + + for (const file of dataFiles) { + await forEachIonItem(s3, file, item => { + if (item.pk !== 'USER' || typeof item.sk !== 'string') { + return; + } + if (item.cleaned === true) { + return; + } + const lastSeen = item.lastSeen; + if (typeof lastSeen !== 'number' || lastSeen >= inactiveBeforeMs) { + return; + } + candidates.add(item.sk); + }); + } + + return [...candidates]; +} diff --git a/crons/src/utils/gameState.test.ts b/crons/src/utils/gameState.test.ts new file mode 100644 index 00000000..fdc70f97 --- /dev/null +++ b/crons/src/utils/gameState.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { gzipSync } from "node:zlib"; +import { GameFactory } from "@abstractplay/gameslib"; +import { decompressGameState, isCompressedGameState } from "./gameState.js"; + +const smallState = '{"game":"saltire","numplayers":2}'; + +describe("decompressGameState", () => { + it("passes through small JSON state unchanged", () => { + expect(decompressGameState(smallState)).toBe(smallState); + expect(isCompressedGameState(smallState)).toBe(false); + }); + + it("decompresses gz-prefixed backend state", () => { + const compressed = "gz:" + gzipSync(Buffer.from(smallState, "utf8")).toString("base64"); + expect(isCompressedGameState(compressed)).toBe(true); + expect(decompressGameState(compressed)).toBe(smallState); + }); + + it("decompresses legacy base64 gzip without prefix", () => { + const legacy = gzipSync(Buffer.from(smallState, "utf8")).toString("base64"); + expect(isCompressedGameState(legacy)).toBe(true); + expect(decompressGameState(legacy)).toBe(smallState); + }); + + it("GameFactory accepts decompressed state", () => { + const engine = GameFactory("archimedes", undefined, ["8x10"]); + expect(engine).toBeDefined(); + const serialized = engine!.serialize(); + const compressed = "gz:" + gzipSync(Buffer.from(serialized, "utf8")).toString("base64"); + const restored = GameFactory("archimedes", decompressGameState(compressed), ["8x10"]); + expect(restored).toBeDefined(); + expect(restored!.metaGame).toBe("archimedes"); + }); +}); diff --git a/crons/src/utils/gameState.ts b/crons/src/utils/gameState.ts new file mode 100644 index 00000000..55e83e05 --- /dev/null +++ b/crons/src/utils/gameState.ts @@ -0,0 +1,87 @@ +import { gzipSync, gunzipSync } from "node:zlib"; + +export const GAME_STATE_COMPRESS_THRESHOLD_BYTES = 300_000; +/** DynamoDB item limit is 400KB; leave headroom for non-state attributes. */ +export const GAME_STATE_MAX_STORED_BYTES = 390_000; + +const COMPRESSED_PREFIX = "gz:"; + +function stateByteLength(state: string): number { + return Buffer.byteLength(state, "utf8"); +} + +function isGzipBuffer(buf: Buffer): boolean { + return buf.length >= 2 && buf[0] === 0x1f && buf[1] === 0x8b; +} + +export function isCompressedGameState(state: string): boolean { + if (!state || state.startsWith("{") || state.startsWith("[")) { + return false; + } + if (state.startsWith(COMPRESSED_PREFIX)) { + return true; + } + try { + const buf = Buffer.from(state, "base64"); + return isGzipBuffer(buf); + } catch { + return false; + } +} + +function gunzipBase64(base64: string): string { + return gunzipSync(Buffer.from(base64, "base64")).toString("utf8"); +} + +export function decompressGameState(state: string): string { + if (!state || state.startsWith("{") || state.startsWith("[")) { + return state; + } + if (state.startsWith(COMPRESSED_PREFIX)) { + return gunzipBase64(state.slice(COMPRESSED_PREFIX.length)); + } + try { + const buf = Buffer.from(state, "base64"); + if (isGzipBuffer(buf)) { + return gunzipSync(buf).toString("utf8"); + } + } catch { + // fall through + } + return state; +} + +function gzipToPrefixedBase64(state: string): string { + const compressed = gzipSync(Buffer.from(state, "utf8")); + return COMPRESSED_PREFIX + compressed.toString("base64"); +} + +function assertStoredStateSize(state: string): void { + const bytes = stateByteLength(state); + if (bytes > GAME_STATE_MAX_STORED_BYTES) { + throw new Error( + `Game state is ${bytes} bytes after compression (limit ${GAME_STATE_MAX_STORED_BYTES}); ` + + 'DynamoDB item would exceed the 400KB limit', + ); + } +} + +export function compressGameStateIfNeeded(state: string): string { + if (isCompressedGameState(state)) { + return state; + } + if (stateByteLength(state) <= GAME_STATE_COMPRESS_THRESHOLD_BYTES) { + return state; + } + const compressed = gzipToPrefixedBase64(state); + assertStoredStateSize(compressed); + return compressed; +} + +export function prepareGameStateForStorage(record: T): T { + const compressed = compressGameStateIfNeeded(record.state); + if (compressed === record.state) { + return record; + } + return { ...record, state: compressed }; +} diff --git a/crons/src/utils/isoToCountryCode.ts b/crons/src/utils/isoToCountryCode.ts new file mode 100644 index 00000000..cdfd1a27 --- /dev/null +++ b/crons/src/utils/isoToCountryCode.ts @@ -0,0 +1,18 @@ +import { countryCodeList } from "./countryCodeList.js"; + +function isoToCountryCode(isoCode: string, keyToGet: 'alpha2' | 'alpha3' | 'numeric' | 'countryName' = 'alpha2'): string|undefined { + if (isoCode !== undefined) { + const entry = countryCodeList.find((countryObj) => ( + countryObj.alpha2 === isoCode + || countryObj.alpha3 === isoCode + || countryObj.numeric === isoCode + )); + if ( (entry !== undefined) && (entry[keyToGet] !== undefined) ) { + return entry[keyToGet]; + } + return undefined; + } + return undefined; +} + +export { isoToCountryCode } diff --git a/crons/src/utils/moveSeasonality.test.ts b/crons/src/utils/moveSeasonality.test.ts new file mode 100644 index 00000000..3c45dc5f --- /dev/null +++ b/crons/src/utils/moveSeasonality.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { computeMoveSeasonality, computeWeeklyActiveMovers, alignWeeklyActiveMovers } from "./moveSeasonality.js"; + +describe("computeMoveSeasonality", () => { + it("bins moves and unique players by UTC day and hour", () => { + const monday = Date.parse("2026-01-05T15:30:00.000Z"); + const mondayLater = Date.parse("2026-01-05T16:00:00.000Z"); + const result = computeMoveSeasonality([ + { player: "a", time: monday }, + { player: "a", time: mondayLater }, + { player: "b", time: mondayLater }, + ], 365); + expect(result.movesByDow[1]).toBe(3); + expect(result.playersByDow[1]).toBe(2); + expect(result.movesByHour[15]).toBe(1); + expect(result.movesByHour[16]).toBe(2); + expect(result.windowDays).toBe(365); + }); +}); + +describe("computeWeeklyActiveMovers", () => { + const weekMs = 7 * 24 * 60 * 60 * 1000; + const originMs = 0; + + it("counts distinct players per seven-day bucket from origin", () => { + const result = computeWeeklyActiveMovers([ + { player: "a", time: 1000 }, + { player: "a", time: 2000 }, + { player: "b", time: weekMs + 1000 }, + ], originMs); + expect(result.originMs).toBe(0); + expect(result.byWeek).toEqual([1, 1]); + }); +}); + +describe("alignWeeklyActiveMovers", () => { + const weekMs = 7 * 24 * 60 * 60 * 1000; + + it("pads to summarize maxBucket when origins match", () => { + expect(alignWeeklyActiveMovers( + { originMs: 0, byWeek: [3, 5] }, + 0, + 3, + )).toEqual([3, 5, 0, 0]); + }); + + it("offsets buckets when origins differ", () => { + expect(alignWeeklyActiveMovers( + { originMs: weekMs, byWeek: [4] }, + 0, + 2, + )).toEqual([0, 4, 0]); + }); +}); diff --git a/crons/src/utils/moveSeasonality.ts b/crons/src/utils/moveSeasonality.ts new file mode 100644 index 00000000..26cbf2ab --- /dev/null +++ b/crons/src/utils/moveSeasonality.ts @@ -0,0 +1,104 @@ +export const MOVE_SEASONALITY_WINDOW_DAYS = 365; + +export type MoveActivityInput = { + player: string; + time: number; +}; + +export type MoveSeasonalityResult = { + movesByDow: number[]; + playersByDow: number[]; + movesByHour: number[]; + windowDays: number; +}; + +export type WeeklyActiveMovers = { + /** Week-bucket origin (ms); matches `oldestRec` epoch used in summarize histograms. */ + originMs: number; + /** Distinct players with ≥1 move per bucket; index 0 = first seven-day period from origin. */ + byWeek: number[]; +}; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; +const MS_PER_WEEK = 7 * MS_PER_DAY; + +/** Bin move timestamps by UTC day-of-week and hour (aggregated across the window). */ +export function computeMoveSeasonality( + moves: MoveActivityInput[], + windowDays: number = MOVE_SEASONALITY_WINDOW_DAYS, +): MoveSeasonalityResult { + const movesByDow = Array.from({ length: 7 }, () => 0); + const movesByHour = Array.from({ length: 24 }, () => 0); + const playersByDowSets: Set[] = Array.from({ length: 7 }, () => new Set()); + + for (const { player, time } of moves) { + const d = new Date(time); + const dow = d.getUTCDay(); + const hour = d.getUTCHours(); + movesByDow[dow]++; + movesByHour[hour]++; + playersByDowSets[dow].add(player); + } + + return { + movesByDow, + playersByDow: playersByDowSets.map((s) => s.size), + movesByHour, + windowDays, + }; +} + +export function computeWeeklyActiveMovers( + moves: MoveActivityInput[], + originMs: number, +): WeeklyActiveMovers { + const bucketPlayers = new Map>(); + let maxBucket = -1; + for (const { player, time } of moves) { + if (time < originMs) { + continue; + } + const bucket = Math.floor((time - originMs) / MS_PER_WEEK); + if (bucket < 0) { + continue; + } + maxBucket = Math.max(maxBucket, bucket); + let set = bucketPlayers.get(bucket); + if (set === undefined) { + set = new Set(); + bucketPlayers.set(bucket, set); + } + set.add(player); + } + const byWeek: number[] = []; + for (let i = 0; i <= maxBucket; i++) { + byWeek.push(bucketPlayers.get(i)?.size ?? 0); + } + return { originMs, byWeek }; +} + +/** Align move-time weekly counts to summarize histogram buckets (same origin and length). */ +export function alignWeeklyActiveMovers( + movers: WeeklyActiveMovers | undefined, + originMs: number, + maxBucket: number, +): number[] { + const aligned = Array.from({ length: maxBucket + 1 }, () => 0); + if (movers === undefined || movers.byWeek.length === 0) { + return aligned; + } + if (movers.originMs === originMs) { + for (let i = 0; i <= maxBucket && i < movers.byWeek.length; i++) { + aligned[i] = movers.byWeek[i] ?? 0; + } + return aligned; + } + const offset = Math.floor((movers.originMs - originMs) / MS_PER_WEEK); + for (let i = 0; i < movers.byWeek.length; i++) { + const target = i + offset; + if (target >= 0 && target <= maxBucket) { + aligned[target] = movers.byWeek[i] ?? 0; + } + } + return aligned; +} diff --git a/crons/src/utils/playerSummaryHash.test.ts b/crons/src/utils/playerSummaryHash.test.ts new file mode 100644 index 00000000..ae0aac0f --- /dev/null +++ b/crons/src/utils/playerSummaryHash.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { playerSummarySliceContentHash, stableJsonHash } from "./playerSummaryHash.js"; + +describe("stableJsonHash", () => { + it("is independent of object key order", () => { + const a = stableJsonHash({ b: 2, a: 1 }); + const b = stableJsonHash({ a: 1, b: 2 }); + expect(a).toBe(b); + }); + + it("produces different hashes for different values", () => { + expect(stableJsonHash({ a: 1 })).not.toBe(stableJsonHash({ a: 2 })); + }); +}); + +describe("playerSummarySliceContentHash", () => { + const baseSlice = { + generated: "2026-01-01T00:00:00.000Z", + user: "alice", + players: { allPlays: 5 }, + histograms: { players: [1, 0] }, + ratings: { highest: [] }, + }; + + it("excludes generated from the hash", () => { + const hashA = playerSummarySliceContentHash({ + ...baseSlice, + generated: "2026-01-01T00:00:00.000Z", + }); + const hashB = playerSummarySliceContentHash({ + ...baseSlice, + generated: "2026-02-02T00:00:00.000Z", + }); + expect(hashA).toBe(hashB); + }); + + it("changes when substantive fields change", () => { + const hashA = playerSummarySliceContentHash(baseSlice); + const hashB = playerSummarySliceContentHash({ + ...baseSlice, + players: { allPlays: 6 }, + }); + expect(hashA).not.toBe(hashB); + }); +}); diff --git a/crons/src/utils/playerSummaryHash.ts b/crons/src/utils/playerSummaryHash.ts new file mode 100644 index 00000000..f073fc8c --- /dev/null +++ b/crons/src/utils/playerSummaryHash.ts @@ -0,0 +1,24 @@ +import { createHash } from "node:crypto"; +import type { PlayerSummarySlice } from "types/stats/StatSummaryTiers.js"; + +function stableStringify(value: unknown): string { + if (value === null || typeof value !== "object") { + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item)).join(",")}]`; + } + const record = value as Record; + const keys = Object.keys(record).sort((a, b) => a.localeCompare(b)); + const parts = keys.map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`); + return `{${parts.join(",")}}`; +} + +export function stableJsonHash(value: unknown): string { + return createHash("sha256").update(stableStringify(value)).digest("hex"); +} + +export function playerSummarySliceContentHash(slice: PlayerSummarySlice): string { + const { generated: _generated, ...content } = slice; + return stableJsonHash(content); +} diff --git a/crons/src/utils/recAnalytics.test.ts b/crons/src/utils/recAnalytics.test.ts new file mode 100644 index 00000000..de325b08 --- /dev/null +++ b/crons/src/utils/recAnalytics.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { + aggregateEvents, + buildAnalyticsSummary, + buildMarkdownReport, + buildRollingSlice, + emptyAnalyticsSlice, + mergeSlices, + parseRecommendationEvent, + type NormalizedRecEvent, + type RawDdbItem, +} from "./recAnalytics.js"; + +function showEvent(overrides: Partial & { batchId: string }): NormalizedRecEvent { + return { + eventTimeMs: 1_700_000_000_000, + event: "rec_show", + surface: "gamePicker", + tier: "warm", + gameIds: ["go", "hex"], + reasons: ["content", "cooccur"], + ...overrides, + }; +} + +function clickEvent(overrides: Partial & { batchId: string }): NormalizedRecEvent { + return { + eventTimeMs: 1_700_000_000_100, + event: "rec_click", + surface: "gamePicker", + tier: "warm", + metaGame: "go", + position: 0, + reasonType: "content", + ...overrides, + }; +} + +function challengeEvent(overrides: Partial & { batchId: string }): NormalizedRecEvent { + return { + eventTimeMs: 1_700_000_000_200, + event: "rec_challenge", + surface: "gamePicker", + tier: "warm", + metaGame: "go", + ...overrides, + }; +} + +describe("aggregateEvents", () => { + it("joins show + click + challenge on the same batchId", () => { + const slice = aggregateEvents([ + showEvent({ batchId: "batch-a" }), + clickEvent({ batchId: "batch-a" }), + challengeEvent({ batchId: "batch-a" }), + ]); + + expect(slice.totals).toEqual({ shows: 1, clicks: 1, challenges: 1 }); + expect(slice.rates.ctr).toBe(1); + expect(slice.rates.challengeRate).toBe(1); + expect(slice.rates.endToEndRate).toBe(1); + expect(slice.topClickedMetaGames).toEqual([{ metaGame: "go", count: 1 }]); + expect(slice.topChallengedMetaGames).toEqual([{ metaGame: "go", count: 1 }]); + }); + + it("computes CTR by reasonType across batches", () => { + const slice = aggregateEvents([ + showEvent({ batchId: "batch-a" }), + clickEvent({ batchId: "batch-a", reasonType: "content" }), + showEvent({ batchId: "batch-b" }), + clickEvent({ batchId: "batch-b", reasonType: "cooccur", metaGame: "hex", position: 1 }), + ]); + + expect(slice.totals.shows).toBe(2); + expect(slice.totals.clicks).toBe(2); + expect(slice.rates.ctr).toBe(1); + expect(slice.byReasonType.content?.clicks).toBe(1); + expect(slice.byReasonType.cooccur?.clicks).toBe(1); + expect(slice.byReasonType.content?.showReasons).toBe(2); + expect(slice.byReasonType.cooccur?.showReasons).toBe(2); + }); + + it("flags orphan clicks without a matching show", () => { + const slice = aggregateEvents([ + clickEvent({ batchId: "orphan-batch" }), + ]); + + expect(slice.totals).toEqual({ shows: 0, clicks: 1, challenges: 0 }); + expect(slice.dataQuality.orphanClicks).toBe(1); + }); + + it("returns zeros for an empty day without throwing", () => { + const slice = aggregateEvents([]); + expect(slice.totals).toEqual({ shows: 0, clicks: 0, challenges: 0 }); + expect(slice.rates).toEqual({ ctr: 0, challengeRate: 0, endToEndRate: 0 }); + expect(slice.dataQuality.eventsProcessed).toBe(0); + }); +}); + +describe("mergeSlices / rolling windows", () => { + it("merges two daily slices into correct 7d totals", () => { + const dayOne = aggregateEvents([ + showEvent({ batchId: "d1-a" }), + clickEvent({ batchId: "d1-a" }), + ]); + const dayTwo = aggregateEvents([ + showEvent({ batchId: "d2-a" }), + showEvent({ batchId: "d2-b" }), + ]); + + const rolling = buildRollingSlice( + [ + { date: "2026-08-11", slice: dayOne }, + { date: "2026-08-12", slice: dayTwo }, + ], + "2026-08-12", + 7, + ); + + expect(rolling.totals).toEqual({ shows: 3, clicks: 1, challenges: 0 }); + expect(rolling.rates.ctr).toBeCloseTo(1 / 3); + }); + + it("buildAnalyticsSummary includes rolling7d and rolling30d", () => { + const windowSlice = aggregateEvents([showEvent({ batchId: "w1" })]); + const daily = [{ date: "2026-08-12", slice: windowSlice }]; + const summary = buildAnalyticsSummary(windowSlice, daily, "2026-08-12"); + + expect(summary.totals.shows).toBe(1); + expect(summary.rolling7d.totals.shows).toBe(1); + expect(summary.rolling30d.totals.shows).toBe(1); + }); + + it("mergeSlices is additive across slices", () => { + const a = aggregateEvents([showEvent({ batchId: "a" })]); + const b = aggregateEvents([showEvent({ batchId: "b" }), clickEvent({ batchId: "b" })]); + const merged = mergeSlices([a, b]); + expect(merged.totals).toEqual({ shows: 2, clicks: 1, challenges: 0 }); + }); +}); + +describe("parseRecommendationEvent", () => { + it("parses a valid rec_show row", () => { + const item: RawDdbItem = { + pk: "RECOMMENDS#user-1", + sk: "1700000000000#abc", + event: "rec_show", + batchId: "batch-1", + surface: "explore", + tier: "cold", + gameIds: ["go"], + reasons: ["content"], + }; + const parsed = parseRecommendationEvent(item); + expect(parsed.ok).toBe(true); + if (parsed.ok) { + expect(parsed.event.batchId).toBe("batch-1"); + expect(parsed.event.eventTimeMs).toBe(1_700_000_000_000); + } + }); +}); + +describe("buildMarkdownReport", () => { + it("includes hybrid weight reference and handles zero shows", () => { + const summary = buildAnalyticsSummary(emptyAnalyticsSlice(), [], "2026-08-12"); + summary.generatedAt = "2026-08-12T03:00:00.000Z"; + const md = buildMarkdownReport({ runDate: "2026-08-12", summary }); + + expect(md).toContain("No recommendation impressions"); + expect(md).toContain("Review notes for agents"); + expect(md).toContain("0.45"); + expect(md).toContain("cooccur"); + }); +}); diff --git a/crons/src/utils/recAnalytics.ts b/crons/src/utils/recAnalytics.ts new file mode 100644 index 00000000..9e994384 --- /dev/null +++ b/crons/src/utils/recAnalytics.ts @@ -0,0 +1,786 @@ +/** Anonymized recommendation impression analytics (funnel / CTR rollups). */ + +export const RECOMMENDS_PK_PREFIX = "RECOMMENDS#"; +export const MIN_RATE_DENOMINATOR = 20; +export const TOP_META_GAMES = 30; +export const MAX_POSITION = 7; +export const WATERMARK_OVERLAP_MS = 5 * 60 * 1000; +export const FIRST_RUN_LOOKBACK_MS = 7 * 24 * 60 * 60 * 1000; +export const DAILY_RETENTION_DAYS = 90; + +/** Reference weights from front `gameRecommendations.js` — for agent review only. */ +export const HYBRID_WEIGHTS_REFERENCE = { + content: 0.45, + cooccur: 0.35, + popularity: 0.15, + recency: 0.1, +} as const; + +export type RecEventType = "rec_show" | "rec_click" | "rec_challenge"; + +export type RawDdbItem = { + pk?: string; + sk?: string; + event?: string; + batchId?: string; + surface?: string; + tier?: string; + metaGame?: string; + position?: number; + reasonType?: string; + gameIds?: string[]; + reasons?: string[]; +}; + +export type NormalizedRecEvent = { + eventTimeMs: number; + event: RecEventType; + batchId: string; + surface: string; + tier: string; + metaGame?: string; + position?: number; + reasonType?: string; + gameIds?: string[]; + reasons?: string[]; +}; + +export type FunnelCounts = { + shows: number; + clicks: number; + challenges: number; +}; + +export type FunnelRates = { + ctr: number; + challengeRate: number; + endToEndRate: number; +}; + +export type DimensionFunnel = FunnelCounts & { + ctr?: number; + challengeRate?: number; + endToEndRate?: number; +}; + +export type ReasonTypeCounts = { + clicks: number; + showReasons: number; +}; + +export type MetaGameCount = { + metaGame: string; + count: number; +}; + +export type DataQuality = { + eventsParsed: number; + eventsSkipped: number; + parseErrors: number; + orphanClicks: number; + duplicateEventsPerBatch: number; + eventsProcessed: number; +}; + +export type AnalyticsSlice = { + window?: { start: string; end: string }; + generatedAt?: string; + totals: FunnelCounts; + rates: FunnelRates; + bySurface: Record; + byTier: Record; + byReasonType: Record; + topClickedMetaGames: MetaGameCount[]; + topChallengedMetaGames: MetaGameCount[]; + positionHistogram: Record; + dataQuality: DataQuality; +}; + +export type AnalyticsSummary = AnalyticsSlice & { + rolling7d: AnalyticsSlice; + rolling30d: AnalyticsSlice; +}; + +export type AnalyticsState = { + lastRunAt: string; + lastSkWatermarkMs: number; + /** `pk#sk` keys already counted in daily rollups (pruned each run). */ + processedKeys?: string[]; +}; + +export type ParseResult = + | { ok: true; event: NormalizedRecEvent } + | { ok: false; reason: string }; + +const REC_EVENT_TYPES = new Set(["rec_show", "rec_click", "rec_challenge"]); + +export function parseSkEpochMs(sk: string): number | null { + const idx = sk.indexOf("#"); + if (idx <= 0) { + return null; + } + const epoch = Number.parseInt(sk.slice(0, idx), 10); + return Number.isFinite(epoch) ? epoch : null; +} + +export function utcDateKey(epochMs: number): string { + return new Date(epochMs).toISOString().slice(0, 10); +} + +export function parseRecommendationEvent(item: RawDdbItem): ParseResult { + if (typeof item.pk !== "string" || !item.pk.startsWith(RECOMMENDS_PK_PREFIX)) { + return { ok: false, reason: "invalid pk" }; + } + if (typeof item.sk !== "string" || item.sk.length === 0) { + return { ok: false, reason: "missing sk" }; + } + const eventTimeMs = parseSkEpochMs(item.sk); + if (eventTimeMs === null) { + return { ok: false, reason: "invalid sk epoch" }; + } + if (typeof item.event !== "string" || !REC_EVENT_TYPES.has(item.event)) { + return { ok: false, reason: "invalid event" }; + } + if (typeof item.batchId !== "string" || item.batchId.trim() === "") { + return { ok: false, reason: "missing batchId" }; + } + if (typeof item.surface !== "string" || item.surface.trim() === "") { + return { ok: false, reason: "missing surface" }; + } + if (typeof item.tier !== "string" || item.tier.trim() === "") { + return { ok: false, reason: "missing tier" }; + } + + const event = item.event as RecEventType; + const normalized: NormalizedRecEvent = { + eventTimeMs, + event, + batchId: item.batchId.trim(), + surface: item.surface.trim(), + tier: item.tier.trim(), + }; + + if (event === "rec_show") { + if (!Array.isArray(item.gameIds) || item.gameIds.length === 0) { + return { ok: false, reason: "rec_show missing gameIds" }; + } + if (!Array.isArray(item.reasons) || item.reasons.length !== item.gameIds.length) { + return { ok: false, reason: "rec_show reasons length mismatch" }; + } + normalized.gameIds = item.gameIds.map((id) => String(id).trim()); + normalized.reasons = item.reasons.map((reason) => String(reason).trim()); + } + + if (event === "rec_click") { + if (typeof item.metaGame !== "string" || item.metaGame.trim() === "") { + return { ok: false, reason: "rec_click missing metaGame" }; + } + if (typeof item.position !== "number" || !Number.isInteger(item.position) || item.position < 0) { + return { ok: false, reason: "rec_click invalid position" }; + } + if (typeof item.reasonType !== "string" || item.reasonType.trim() === "") { + return { ok: false, reason: "rec_click missing reasonType" }; + } + normalized.metaGame = item.metaGame.trim(); + normalized.position = item.position; + normalized.reasonType = item.reasonType.trim(); + } + + if (event === "rec_challenge") { + if (typeof item.metaGame !== "string" || item.metaGame.trim() === "") { + return { ok: false, reason: "rec_challenge missing metaGame" }; + } + normalized.metaGame = item.metaGame.trim(); + } + + return { ok: true, event: normalized }; +} + +function emptyFunnelCounts(): FunnelCounts { + return { shows: 0, clicks: 0, challenges: 0 }; +} + +function emptyDataQuality(): DataQuality { + return { + eventsParsed: 0, + eventsSkipped: 0, + parseErrors: 0, + orphanClicks: 0, + duplicateEventsPerBatch: 0, + eventsProcessed: 0, + }; +} + +export function emptyAnalyticsSlice(): AnalyticsSlice { + return { + totals: emptyFunnelCounts(), + rates: { ctr: 0, challengeRate: 0, endToEndRate: 0 }, + bySurface: {}, + byTier: {}, + byReasonType: {}, + topClickedMetaGames: [], + topChallengedMetaGames: [], + positionHistogram: {}, + dataQuality: emptyDataQuality(), + }; +} + +export function computeRates(totals: FunnelCounts): FunnelRates { + return { + ctr: totals.shows > 0 ? totals.clicks / totals.shows : 0, + challengeRate: totals.clicks > 0 ? totals.challenges / totals.clicks : 0, + endToEndRate: totals.shows > 0 ? totals.challenges / totals.shows : 0, + }; +} + +function getOrCreateDimension( + map: Record, + key: string, +): DimensionFunnel { + const existing = map[key]; + if (existing !== undefined) { + return existing; + } + const created: DimensionFunnel = { shows: 0, clicks: 0, challenges: 0 }; + map[key] = created; + return created; +} + +function getOrCreateReasonType(map: Record, key: string): ReasonTypeCounts { + const existing = map[key]; + if (existing !== undefined) { + return existing; + } + const created: ReasonTypeCounts = { clicks: 0, showReasons: 0 }; + map[key] = created; + return created; +} + +function applyDimensionRates(map: Record): void { + for (const slice of Object.values(map)) { + if (slice.shows >= MIN_RATE_DENOMINATOR) { + slice.ctr = slice.clicks / slice.shows; + slice.endToEndRate = slice.challenges / slice.shows; + } + if (slice.clicks >= MIN_RATE_DENOMINATOR) { + slice.challengeRate = slice.challenges / slice.clicks; + } + } +} + +function topMetaGames(counts: Map): MetaGameCount[] { + return [...counts.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, TOP_META_GAMES) + .map(([metaGame, count]) => ({ metaGame, count })); +} + +type BatchAccumulator = { + showCount: number; + clickCount: number; + challengeCount: number; + showSurface?: string; + showTier?: string; + showReasons: string[]; + clickSurfaces: string[]; + clickTiers: string[]; + clickReasonTypes: string[]; + clickMetaGames: string[]; + clickPositions: number[]; + challengeSurfaces: string[]; + challengeTiers: string[]; + challengeMetaGames: string[]; +}; + +function getOrCreateBatch(map: Map, batchId: string): BatchAccumulator { + const existing = map.get(batchId); + if (existing !== undefined) { + return existing; + } + const created: BatchAccumulator = { + showCount: 0, + clickCount: 0, + challengeCount: 0, + showReasons: [], + clickSurfaces: [], + clickTiers: [], + clickReasonTypes: [], + clickMetaGames: [], + clickPositions: [], + challengeSurfaces: [], + challengeTiers: [], + challengeMetaGames: [], + }; + map.set(batchId, created); + return created; +} + +export function aggregateEvents(events: NormalizedRecEvent[]): AnalyticsSlice { + const slice = emptyAnalyticsSlice(); + const batches = new Map(); + let duplicateEventsPerBatch = 0; + + for (const event of events) { + slice.dataQuality.eventsParsed += 1; + const batch = getOrCreateBatch(batches, event.batchId); + + if (event.event === "rec_show") { + if (batch.showCount > 0) { + duplicateEventsPerBatch += 1; + } + batch.showCount += 1; + batch.showSurface = event.surface; + batch.showTier = event.tier; + if (event.reasons !== undefined) { + batch.showReasons.push(...event.reasons); + } + } else if (event.event === "rec_click") { + if (batch.clickCount > 0) { + duplicateEventsPerBatch += 1; + } + batch.clickCount += 1; + batch.clickSurfaces.push(event.surface); + batch.clickTiers.push(event.tier); + if (event.reasonType !== undefined) { + batch.clickReasonTypes.push(event.reasonType); + } + if (event.metaGame !== undefined) { + batch.clickMetaGames.push(event.metaGame); + } + if (event.position !== undefined) { + batch.clickPositions.push(event.position); + } + } else if (event.event === "rec_challenge") { + if (batch.challengeCount > 0) { + duplicateEventsPerBatch += 1; + } + batch.challengeCount += 1; + batch.challengeSurfaces.push(event.surface); + batch.challengeTiers.push(event.tier); + if (event.metaGame !== undefined) { + batch.challengeMetaGames.push(event.metaGame); + } + } + } + + const clickedMetaGames = new Map(); + const challengedMetaGames = new Map(); + let orphanClicks = 0; + + for (const batch of batches.values()) { + if (batch.showCount > 0) { + slice.totals.shows += 1; + const surface = batch.showSurface ?? "unknown"; + const tier = batch.showTier ?? "unknown"; + getOrCreateDimension(slice.bySurface, surface).shows += 1; + getOrCreateDimension(slice.byTier, tier).shows += 1; + for (const reason of batch.showReasons) { + getOrCreateReasonType(slice.byReasonType, reason).showReasons += 1; + } + } + + slice.totals.clicks += batch.clickCount; + slice.totals.challenges += batch.challengeCount; + + if (batch.clickCount > 0 && batch.showCount === 0) { + orphanClicks += batch.clickCount; + } + + for (const surface of batch.clickSurfaces) { + getOrCreateDimension(slice.bySurface, surface).clicks += 1; + } + for (const tier of batch.clickTiers) { + getOrCreateDimension(slice.byTier, tier).clicks += 1; + } + for (const reasonType of batch.clickReasonTypes) { + getOrCreateReasonType(slice.byReasonType, reasonType).clicks += 1; + } + for (const surface of batch.challengeSurfaces) { + getOrCreateDimension(slice.bySurface, surface).challenges += 1; + } + for (const tier of batch.challengeTiers) { + getOrCreateDimension(slice.byTier, tier).challenges += 1; + } + for (const metaGame of batch.clickMetaGames) { + clickedMetaGames.set(metaGame, (clickedMetaGames.get(metaGame) ?? 0) + 1); + } + for (const metaGame of batch.challengeMetaGames) { + challengedMetaGames.set(metaGame, (challengedMetaGames.get(metaGame) ?? 0) + 1); + } + for (const position of batch.clickPositions) { + if (position <= MAX_POSITION) { + const key = String(position); + slice.positionHistogram[key] = (slice.positionHistogram[key] ?? 0) + 1; + } + } + } + + slice.dataQuality.orphanClicks = orphanClicks; + slice.dataQuality.duplicateEventsPerBatch = duplicateEventsPerBatch; + slice.dataQuality.eventsProcessed = events.length; + slice.rates = computeRates(slice.totals); + applyDimensionRates(slice.bySurface); + applyDimensionRates(slice.byTier); + slice.topClickedMetaGames = topMetaGames(clickedMetaGames); + slice.topChallengedMetaGames = topMetaGames(challengedMetaGames); + + return slice; +} + +function addFunnelCounts(target: FunnelCounts, source: FunnelCounts): void { + target.shows += source.shows; + target.clicks += source.clicks; + target.challenges += source.challenges; +} + +function mergeDimensionMaps( + target: Record, + source: Record, +): void { + for (const [key, value] of Object.entries(source)) { + const dim = getOrCreateDimension(target, key); + addFunnelCounts(dim, value); + } +} + +function mergeReasonTypeMaps( + target: Record, + source: Record, +): void { + for (const [key, value] of Object.entries(source)) { + const reason = getOrCreateReasonType(target, key); + reason.clicks += value.clicks; + reason.showReasons += value.showReasons; + } +} + +function mergeMetaGameCounts(target: Map, source: MetaGameCount[]): void { + for (const { metaGame, count } of source) { + target.set(metaGame, (target.get(metaGame) ?? 0) + count); + } +} + +function mergeHistogram( + target: Record, + source: Record, +): void { + for (const [key, count] of Object.entries(source)) { + target[key] = (target[key] ?? 0) + count; + } +} + +function mergeDataQuality(target: DataQuality, source: DataQuality): void { + target.eventsParsed += source.eventsParsed; + target.eventsSkipped += source.eventsSkipped; + target.parseErrors += source.parseErrors; + target.orphanClicks += source.orphanClicks; + target.duplicateEventsPerBatch += source.duplicateEventsPerBatch; + target.eventsProcessed += source.eventsProcessed; +} + +export function mergeSlices(slices: AnalyticsSlice[]): AnalyticsSlice { + if (slices.length === 0) { + return emptyAnalyticsSlice(); + } + + const merged = emptyAnalyticsSlice(); + const clickedMetaGames = new Map(); + const challengedMetaGames = new Map(); + + for (const slice of slices) { + addFunnelCounts(merged.totals, slice.totals); + mergeDimensionMaps(merged.bySurface, slice.bySurface); + mergeDimensionMaps(merged.byTier, slice.byTier); + mergeReasonTypeMaps(merged.byReasonType, slice.byReasonType); + mergeMetaGameCounts(clickedMetaGames, slice.topClickedMetaGames); + mergeMetaGameCounts(challengedMetaGames, slice.topChallengedMetaGames); + mergeHistogram(merged.positionHistogram, slice.positionHistogram); + mergeDataQuality(merged.dataQuality, slice.dataQuality); + } + + merged.rates = computeRates(merged.totals); + applyDimensionRates(merged.bySurface); + applyDimensionRates(merged.byTier); + merged.topClickedMetaGames = topMetaGames(clickedMetaGames); + merged.topChallengedMetaGames = topMetaGames(challengedMetaGames); + + return merged; +} + +export function buildRollingSlice( + dailySlices: Array<{ date: string; slice: AnalyticsSlice }>, + asOfDate: string, + windowDays: number, +): AnalyticsSlice { + const startMs = Date.parse(`${asOfDate}T00:00:00.000Z`) - (windowDays - 1) * 86_400_000; + const startDate = utcDateKey(startMs); + const selected = dailySlices + .filter(({ date }) => date >= startDate && date <= asOfDate) + .map(({ slice }) => slice); + return mergeSlices(selected); +} + +export function buildAnalyticsSummary( + windowSlice: AnalyticsSlice, + dailySlices: Array<{ date: string; slice: AnalyticsSlice }>, + asOfDate: string, +): AnalyticsSummary { + return { + ...windowSlice, + rolling7d: buildRollingSlice(dailySlices, asOfDate, 7), + rolling30d: buildRollingSlice(dailySlices, asOfDate, 30), + }; +} + +export function ingestRawItems(items: RawDdbItem[]): { + events: NormalizedRecEvent[]; + dataQuality: Pick; +} { + const events: NormalizedRecEvent[] = []; + let eventsSkipped = 0; + let parseErrors = 0; + + for (const item of items) { + const parsed = parseRecommendationEvent(item); + if (!parsed.ok) { + parseErrors += 1; + continue; + } + events.push(parsed.event); + } + + eventsSkipped = items.length - events.length - parseErrors; + + return { events, dataQuality: { eventsSkipped, parseErrors } }; +} + +export function itemDedupeKey(item: RawDdbItem): string | null { + if (typeof item.pk !== "string" || typeof item.sk !== "string") { + return null; + } + return `${item.pk}::${item.sk}`; +} + +export function skFromDedupeKey(key: string): string | null { + const idx = key.indexOf("::"); + if (idx < 0) { + return null; + } + return key.slice(idx + 2); +} + +export function pruneProcessedKeys( + keys: string[], + minSkEpochMs: number, +): string[] { + return keys.filter((key) => { + const sk = skFromDedupeKey(key); + if (sk === null) { + return false; + } + const epoch = parseSkEpochMs(sk); + return epoch !== null && epoch >= minSkEpochMs; + }); +} + +function formatRate(rate: number): string { + return `${(rate * 100).toFixed(1)}%`; +} + +function formatDelta(current: number, previous: number): string { + if (previous === 0) { + return current === 0 ? "—" : "+∞"; + } + const delta = ((current - previous) / previous) * 100; + const sign = delta > 0 ? "+" : ""; + return `${sign}${delta.toFixed(1)}%`; +} + +function markdownTable(headers: string[], rows: string[][]): string { + const headerRow = `| ${headers.join(" | ")} |`; + const separator = `| ${headers.map(() => "---").join(" | ")} |`; + const body = rows.map((row) => `| ${row.join(" | ")} |`).join("\n"); + return [headerRow, separator, body].filter((line) => line.length > 0).join("\n"); +} + +export type MarkdownReportInput = { + runDate: string; + summary: AnalyticsSummary; + priorWeek?: AnalyticsSlice; +}; + +export function buildMarkdownReport({ runDate, summary, priorWeek }: MarkdownReportInput): string { + const lines: string[] = [ + `# Recommendation analytics — ${runDate}`, + "", + `Generated: ${summary.generatedAt ?? "unknown"}`, + "", + ]; + + if (summary.window !== undefined) { + lines.push( + `Window: ${summary.window.start} → ${summary.window.end}`, + "", + ); + } + + if (summary.totals.shows === 0) { + lines.push( + "## No recommendation impressions", + "", + "No `rec_show` events were recorded in this window. CTR and funnel metrics are not meaningful.", + "", + ); + } + + lines.push( + "## Funnel (window)", + "", + markdownTable( + ["Metric", "Value"], + [ + ["Shows", String(summary.totals.shows)], + ["Clicks", String(summary.totals.clicks)], + ["Challenges", String(summary.totals.challenges)], + ["CTR", formatRate(summary.rates.ctr)], + ["Challenge rate", formatRate(summary.rates.challengeRate)], + ["End-to-end rate", formatRate(summary.rates.endToEndRate)], + ], + ), + "", + ); + + const surfaceRows = Object.entries(summary.bySurface) + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([surface, dim]) => [ + surface, + String(dim.shows), + String(dim.clicks), + dim.ctr !== undefined ? formatRate(dim.ctr) : "—", + ]); + if (surfaceRows.length > 0) { + lines.push( + "## By surface", + "", + markdownTable(["Surface", "Shows", "Clicks", "CTR"], surfaceRows), + "", + ); + } + + const tierRows = Object.entries(summary.byTier) + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([tier, dim]) => [ + tier, + String(dim.shows), + String(dim.clicks), + dim.ctr !== undefined ? formatRate(dim.ctr) : "—", + ]); + if (tierRows.length > 0) { + lines.push( + "## By tier", + "", + markdownTable(["Tier", "Shows", "Clicks", "CTR"], tierRows), + "", + ); + } + + const reasonRows = Object.entries(summary.byReasonType) + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([reason, counts]) => [ + reason, + String(counts.showReasons), + String(counts.clicks), + ]); + if (reasonRows.length > 0) { + lines.push( + "## By reason type", + "", + markdownTable(["Reason", "Show slots", "Clicks"], reasonRows), + "", + ); + } + + if (summary.topClickedMetaGames.length > 0) { + lines.push( + "## Top clicked meta-games (window)", + "", + markdownTable( + ["Meta-game", "Clicks"], + summary.topClickedMetaGames.map(({ metaGame, count }) => [metaGame, String(count)]), + ), + "", + ); + } + + if (summary.topChallengedMetaGames.length > 0) { + lines.push( + "## Top challenged meta-games (window)", + "", + markdownTable( + ["Meta-game", "Challenges"], + summary.topChallengedMetaGames.map(({ metaGame, count }) => [metaGame, String(count)]), + ), + "", + ); + } + + const rolling = summary.rolling7d; + const prior = priorWeek?.totals; + if (prior !== undefined) { + lines.push( + "## Week-over-week (rolling 7d vs prior 7d)", + "", + markdownTable( + ["Metric", "Current 7d", "Prior 7d", "Delta"], + [ + ["Shows", String(rolling.totals.shows), String(prior.shows), formatDelta(rolling.totals.shows, prior.shows)], + ["Clicks", String(rolling.totals.clicks), String(prior.clicks), formatDelta(rolling.totals.clicks, prior.clicks)], + ["CTR", formatRate(rolling.rates.ctr), formatRate(computeRates(prior).ctr), formatDelta(rolling.rates.ctr, computeRates(prior).ctr)], + ], + ), + "", + ); + } else { + lines.push( + "## Rolling 7d", + "", + markdownTable( + ["Metric", "Value"], + [ + ["Shows", String(rolling.totals.shows)], + ["Clicks", String(rolling.totals.clicks)], + ["CTR", formatRate(rolling.rates.ctr)], + ], + ), + "", + ); + } + + lines.push( + "## Data quality", + "", + markdownTable( + ["Metric", "Value"], + [ + ["Events processed", String(summary.dataQuality.eventsProcessed)], + ["Parse errors", String(summary.dataQuality.parseErrors)], + ["Orphan clicks", String(summary.dataQuality.orphanClicks)], + ["Duplicate events per batch", String(summary.dataQuality.duplicateEventsPerBatch)], + ], + ), + "", + "## Review notes for agents", + "", + "Current hybrid warm-tier weights in the front-end recommender (reference only — not tuned from this job):", + "", + markdownTable( + ["Signal", "Weight"], + Object.entries(HYBRID_WEIGHTS_REFERENCE).map(([signal, weight]) => [signal, String(weight)]), + ), + "", + "Live recommender does **not** read these analytics artifacts. Use metrics here to inform future weight or product decisions manually.", + "", + ); + + return lines.join("\n"); +} diff --git a/crons/src/utils/recordGameId.test.ts b/crons/src/utils/recordGameId.test.ts new file mode 100644 index 00000000..d649bdea --- /dev/null +++ b/crons/src/utils/recordGameId.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { + encodeRecordGameId, + parseRecordGameId, + variantComboKey, +} from "./recordGameId.js"; + +const UUID = "f47ac10b-58cc-4372-a567-0e02b2c3d479"; + +describe("variantComboKey", () => { + it("returns empty string for no variants", () => { + expect(variantComboKey([])).toBe(""); + }); + + it("sorts and joins variant UIDs", () => { + expect(variantComboKey(["handicap", "9x9"])).toBe("9x9|handicap"); + }); +}); + +describe("encodeRecordGameId", () => { + it("encodes with sorted variant UIDs", () => { + expect(encodeRecordGameId(UUID, "go", ["handicap", "9x9"])).toBe( + `${UUID}#go:9x9|handicap`, + ); + }); + + it("encodes with trailing colon when no variants", () => { + expect(encodeRecordGameId(UUID, "chess", [])).toBe(`${UUID}#chess:`); + }); + + it("canonicalizes unsorted input", () => { + const a = encodeRecordGameId(UUID, "go", ["b", "a"]); + const b = encodeRecordGameId(UUID, "go", ["a", "b"]); + expect(a).toBe(b); + expect(a).toBe(`${UUID}#go:a|b`); + }); +}); + +describe("parseRecordGameId", () => { + it("round-trips encoded ids with variants", () => { + const encoded = encodeRecordGameId(UUID, "go", ["9x9", "handicap"]); + expect(parseRecordGameId(encoded)).toEqual({ + instanceId: UUID, + metaGame: "go", + variantUids: ["9x9", "handicap"], + legacy: false, + }); + }); + + it("round-trips encoded ids without variants", () => { + const encoded = encodeRecordGameId(UUID, "chess", []); + expect(parseRecordGameId(encoded)).toEqual({ + instanceId: UUID, + metaGame: "chess", + variantUids: [], + legacy: false, + }); + }); + + it("parses legacy metaGame#uuid format", () => { + expect(parseRecordGameId(`go#${UUID}`)).toEqual({ + instanceId: UUID, + metaGame: "go", + variantUids: [], + legacy: true, + }); + }); + + it("sorts variant UIDs from encoded ids", () => { + const parsed = parseRecordGameId(`${UUID}#go:handicap|9x9`); + expect(parsed?.variantUids).toEqual(["9x9", "handicap"]); + }); + + it("returns undefined for empty or malformed ids", () => { + expect(parseRecordGameId("")).toBeUndefined(); + expect(parseRecordGameId("legacy-1")).toBeUndefined(); + expect(parseRecordGameId("not-a-uuid#go:")).toBeUndefined(); + expect(parseRecordGameId(`go#not-a-uuid`)).toBeUndefined(); + expect(parseRecordGameId(`${UUID}#:`)).toBeUndefined(); + }); +}); diff --git a/crons/src/utils/recordGameId.ts b/crons/src/utils/recordGameId.ts new file mode 100644 index 00000000..5fad11a2 --- /dev/null +++ b/crons/src/utils/recordGameId.ts @@ -0,0 +1,75 @@ +/** Encode/decode stable metaGame + variant UIDs in `header.site.gameid`. */ + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export type ParsedRecordGameId = { + instanceId: string; + metaGame: string; + /** Canonical sorted variant UIDs (empty when none). */ + variantUids: string[]; + legacy: boolean; +}; + +/** Sorted variant UIDs joined with `|`, or `""` when none. */ +export function variantComboKey(variantUids: readonly string[]): string { + if (variantUids.length === 0) { + return ""; + } + return [...variantUids].sort().join("|"); +} + +/** `{instanceId}#{metaGame}:{sortedVariantUids}` — trailing colon when no variants. */ +export function encodeRecordGameId( + instanceId: string, + metaGame: string, + variantUids: readonly string[], +): string { + return `${instanceId}#${metaGame}:${variantComboKey(variantUids)}`; +} + +export function parseRecordGameId(gameid: string): ParsedRecordGameId | undefined { + if (gameid.length === 0) { + return undefined; + } + + const colonIdx = gameid.indexOf(":"); + if (colonIdx !== -1) { + const prefix = gameid.slice(0, colonIdx); + const hashIdx = prefix.indexOf("#"); + if (hashIdx === -1) { + return undefined; + } + const instanceId = prefix.slice(0, hashIdx); + const metaGame = prefix.slice(hashIdx + 1); + if (!UUID_RE.test(instanceId) || metaGame.length === 0) { + return undefined; + } + const variantPart = gameid.slice(colonIdx + 1); + const variantUids = + variantPart.length === 0 + ? [] + : variantPart.split("|").filter((v) => v.length > 0); + return { + instanceId, + metaGame, + variantUids: [...variantUids].sort(), + legacy: false, + }; + } + + const hashIdx = gameid.indexOf("#"); + if (hashIdx === -1) { + return undefined; + } + const metaGame = gameid.slice(0, hashIdx); + const instanceId = gameid.slice(hashIdx + 1); + if (metaGame.length === 0 || !UUID_RE.test(instanceId)) { + return undefined; + } + return { + instanceId, + metaGame, + variantUids: [], + legacy: true, + }; +} diff --git a/crons/src/utils/recordTournament.test.ts b/crons/src/utils/recordTournament.test.ts new file mode 100644 index 00000000..e51a263c --- /dev/null +++ b/crons/src/utils/recordTournament.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; +import type { Tournament } from "types/index.js"; +import { findTournamentForGame } from "./recordTournament.js"; + +const TOURNAMENT_ID = "ca7cc52c-966c-48a3-8237-5195ea7c84ac"; + +function makeTournament(overrides: Partial & Pick): Tournament { + return { + metaGame: "zola", + variants: [], + number: 42, + started: true, + dateCreated: 0, + datePreviousEnded: 0, + ...overrides, + }; +} + +describe("findTournamentForGame", () => { + const active = makeTournament({ + pk: "TOURNAMENT", + sk: TOURNAMENT_ID, + id: TOURNAMENT_ID, + }); + const archived = makeTournament({ + pk: "COMPLETEDTOURNAMENT", + sk: `zola#${TOURNAMENT_ID}`, + id: TOURNAMENT_ID, + }); + + it("matches an active tournament by plain uuid", () => { + expect(findTournamentForGame([active, archived], TOURNAMENT_ID, "zola")).toBe(active); + }); + + it("matches an archived tournament when the game stores metaGame#uuid", () => { + expect(findTournamentForGame([active, archived], `zola#${TOURNAMENT_ID}`, "zola")).toBe(archived); + }); + + it("matches an archived tournament when the game stores plain uuid", () => { + expect(findTournamentForGame([archived], TOURNAMENT_ID, "zola")).toBe(archived); + }); + + it("returns undefined when no tournament row matches", () => { + expect(findTournamentForGame([archived], "missing-id", "zola")).toBeUndefined(); + }); +}); diff --git a/crons/src/utils/recordTournament.ts b/crons/src/utils/recordTournament.ts new file mode 100644 index 00000000..75c5cf66 --- /dev/null +++ b/crons/src/utils/recordTournament.ts @@ -0,0 +1,22 @@ +import type { Tournament } from "types/index.js"; + +/** Resolve a game `tournament` field to the matching dump row (active or archived). */ +export function findTournamentForGame( + tournaments: Tournament[], + tournamentRef: string, + metaGame: string, +): Tournament | undefined { + const direct = tournaments.find(t => t.id === tournamentRef || t.sk === tournamentRef); + if (direct !== undefined) { + return direct; + } + + const uuidFromRef = tournamentRef.includes("#") + ? tournamentRef.slice(tournamentRef.indexOf("#") + 1) + : tournamentRef; + + return tournaments.find(t => + t.id === uuidFromRef || + t.sk === `${metaGame}#${uuidFromRef}` + ); +} diff --git a/crons/src/utils/recordUnrated.test.ts b/crons/src/utils/recordUnrated.test.ts new file mode 100644 index 00000000..81f56294 --- /dev/null +++ b/crons/src/utils/recordUnrated.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { gameRecordIsUnrated, hasUnratedVariant } from "./recordUnrated.js"; + +describe("hasUnratedVariant", () => { + it("returns true for arimaa free placement variant", () => { + expect(hasUnratedVariant("arimaa", ["free"])).toBe(true); + }); + + it("returns false for a normal variant combo", () => { + expect(hasUnratedVariant("archimedes", ["8x10"])).toBe(false); + }); + + it("returns false for unknown meta game", () => { + expect(hasUnratedVariant("not-a-real-game", ["free"])).toBe(false); + }); +}); + +describe("gameRecordIsUnrated", () => { + it("returns false when rated is true and variants are not unrated", () => { + expect(gameRecordIsUnrated("archimedes", ["8x10"], true)).toBe(false); + }); + + it("returns true when rated is false", () => { + expect(gameRecordIsUnrated("archimedes", ["8x10"], false)).toBe(true); + }); + + it("returns true when rated is missing", () => { + expect(gameRecordIsUnrated("archimedes", ["8x10"], undefined)).toBe(true); + }); + + it("returns true when rated is true but a variant forces unrated", () => { + expect(gameRecordIsUnrated("arimaa", ["free"], true)).toBe(true); + }); +}); diff --git a/crons/src/utils/recordUnrated.ts b/crons/src/utils/recordUnrated.ts new file mode 100644 index 00000000..e13371ec --- /dev/null +++ b/crons/src/utils/recordUnrated.ts @@ -0,0 +1,23 @@ +import { gameinfo } from "@abstractplay/gameslib"; + +export function hasUnratedVariant(metaGame: string, variantUids: string[]): boolean { + const variants = gameinfo.get(metaGame)?.variants; + if (variants === undefined) { + return false; + } + const byUid = new Map(variants.map((v) => [v.uid, v])); + for (const uid of variantUids) { + if (byUid.get(uid)?.unrated === true) { + return true; + } + } + return false; +} + +export function gameRecordIsUnrated( + metaGame: string, + variantUids: string[], + rated?: boolean, +): boolean { + return rated !== true || hasUnratedVariant(metaGame, variantUids); +} diff --git a/crons/src/utils/recordsJson.test.ts b/crons/src/utils/recordsJson.test.ts new file mode 100644 index 00000000..9ebcffa7 --- /dev/null +++ b/crons/src/utils/recordsJson.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { + RECORDS_JSON_CACHE_CONTROL, + RECORDS_MANIFEST_CACHE_CONTROL, + buildRecordsJsonPutInput, +} from "./recordsJson.js"; + +describe("recordsJson", () => { + it("sets application/json and default cache control for batch JSON", () => { + const input = buildRecordsJsonPutInput("_summary.json", { ok: true }); + expect(input.ContentType).toBe("application/json"); + expect(input.CacheControl).toBe(RECORDS_JSON_CACHE_CONTROL); + expect(input.CacheControl).toBe("public, max-age=0, must-revalidate"); + }); + + it("allows manifest-specific cache control", () => { + const input = buildRecordsJsonPutInput("_manifest.json", {}, { + cacheControl: RECORDS_MANIFEST_CACHE_CONTROL, + }); + expect(input.CacheControl).toBe("no-cache"); + }); +}); diff --git a/crons/src/utils/recordsJson.ts b/crons/src/utils/recordsJson.ts new file mode 100644 index 00000000..b03e8402 --- /dev/null +++ b/crons/src/utils/recordsJson.ts @@ -0,0 +1,83 @@ +import { GetObjectCommand, PutObjectCommand, type PutObjectCommandInput, S3Client } from "@aws-sdk/client-s3"; +import { REC_BUCKET } from "../constants/recordsBucket.js"; + +/** Daily batch JSON — revalidate with S3 after each cron overwrite (no blanket invalidation). */ +export const RECORDS_JSON_CACHE_CONTROL = "public, max-age=0, must-revalidate"; + +/** Manifest index — always revalidate before use. */ +export const RECORDS_MANIFEST_CACHE_CONTROL = "no-cache"; + +export type PutRecordsJsonOptions = { + cacheControl?: string; +}; + +export type GetRecordsJsonResult = { + data: T; + bytes: number; +}; + +export function buildRecordsJsonPutInput( + key: string, + body: unknown, + options?: PutRecordsJsonOptions, +): PutObjectCommandInput { + return { + Bucket: REC_BUCKET, + Key: key, + Body: JSON.stringify(body), + ContentType: "application/json", + CacheControl: options?.cacheControl ?? RECORDS_JSON_CACHE_CONTROL, + }; +} + +export async function getRecordsJson( + s3: S3Client, + key: string, +): Promise> { + const response = await s3.send(new GetObjectCommand({ + Bucket: REC_BUCKET, + Key: key, + })); + const body = await response.Body?.transformToString(); + if (body === undefined) { + throw new Error(`Unable to load s3://${REC_BUCKET}/${key}`); + } + return { + data: JSON.parse(body) as T, + bytes: Buffer.byteLength(body, "utf8"), + }; +} + +export async function tryGetRecordsJson( + s3: S3Client, + key: string, +): Promise | undefined> { + try { + return await getRecordsJson(s3, key); + } catch (error) { + const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode; + const name = error instanceof Error ? error.name : ""; + if (status === 404 || name === "NoSuchKey") { + return undefined; + } + throw error; + } +} + +export async function putRecordsJson( + s3: S3Client, + key: string, + body: unknown, + options?: PutRecordsJsonOptions, +): Promise { + const json = JSON.stringify(body); + const response = await s3.send(new PutObjectCommand({ + ...buildRecordsJsonPutInput(key, body, options), + Body: json, + })); + const status = response.$metadata.httpStatusCode; + if (status !== 200) { + throw new Error(`PutObject failed for ${key}: HTTP ${status}`); + } + return Buffer.byteLength(json, "utf8"); +} diff --git a/crons/src/utils/recordsManifest.test.ts b/crons/src/utils/recordsManifest.test.ts new file mode 100644 index 00000000..5f139049 --- /dev/null +++ b/crons/src/utils/recordsManifest.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from "vitest"; +import type { _Object } from "@aws-sdk/client-s3"; +import { buildRecordsManifest, REQUIRED_SUMMARY_KEYS } from "./recordsManifest.js"; +import { + PLAYER_SUMMARY_MANIFEST_KEY, + SUMMARY_MONOLITH_KEY, + SUMMARY_PLAYERS_KEY, + SUMMARY_RATINGS_KEY, + SUMMARY_SITE_KEY, +} from "../constants/recordsBucket.js"; + +const obj = (key: string, size: number): _Object => ({ + Key: key, + Size: size, + LastModified: new Date("2026-01-02T07:30:00.000Z"), +}); + +describe("buildRecordsManifest", () => { + it("wraps bucket listing with summaryFiles entries", () => { + const contents: _Object[] = [ + obj(SUMMARY_MONOLITH_KEY, 4_000_000), + obj(SUMMARY_SITE_KEY, 330_000), + obj(SUMMARY_PLAYERS_KEY, 1_100_000), + obj(SUMMARY_RATINGS_KEY, 2_400_000), + obj(PLAYER_SUMMARY_MANIFEST_KEY, 120), + obj("player/alice-summary.json", 4_000), + ]; + const manifest = buildRecordsManifest(contents, "2026-01-02T07:30:00.000Z"); + expect(manifest.version).toBe(2); + expect(manifest.summaryFiles.monolith).toEqual({ + key: SUMMARY_MONOLITH_KEY, + lastModified: "2026-01-02T07:30:00.000Z", + size: 4_000_000, + }); + expect(manifest.summaryFiles.playerSummaryPattern).toBe("player/{userId}-summary.json"); + expect(manifest.objects).toHaveLength(6); + }); + + it("warns when required summary keys are missing", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + buildRecordsManifest([obj("ALL.json", 1)], "2026-01-02T07:30:00.000Z"); + for (const key of REQUIRED_SUMMARY_KEYS) { + expect(warn).toHaveBeenCalledWith(`Missing summary key in bucket listing: ${key}`); + } + expect(warn).toHaveBeenCalledWith( + `Missing player summary manifest in bucket listing: ${PLAYER_SUMMARY_MANIFEST_KEY}`, + ); + warn.mockRestore(); + }); +}); diff --git a/crons/src/utils/recordsManifest.ts b/crons/src/utils/recordsManifest.ts new file mode 100644 index 00000000..e8c4be6b --- /dev/null +++ b/crons/src/utils/recordsManifest.ts @@ -0,0 +1,77 @@ +import type { _Object } from "@aws-sdk/client-s3"; +import { + PLAYER_SUMMARY_MANIFEST_KEY, + PLAYER_SUMMARY_KEY_PATTERN, + SUMMARY_MONOLITH_KEY, + SUMMARY_PLAYERS_KEY, + SUMMARY_RATINGS_KEY, + SUMMARY_SITE_KEY, +} from "../constants/recordsBucket.js"; + +export type SummaryFileEntry = { + key: string; + lastModified?: string; + size?: number; +}; + +export type RecordsManifestV2 = { + version: 2; + generated: string; + summaryFiles: { + monolith: SummaryFileEntry; + site: SummaryFileEntry; + players: SummaryFileEntry; + ratings: SummaryFileEntry; + playerSummaryPattern: string; + playerManifest: SummaryFileEntry; + }; + objects: _Object[]; +}; + +export const REQUIRED_SUMMARY_KEYS = [ + SUMMARY_MONOLITH_KEY, + SUMMARY_SITE_KEY, + SUMMARY_PLAYERS_KEY, + SUMMARY_RATINGS_KEY, +] as const; + +const summaryEntry = (objectsByKey: Map, key: string): SummaryFileEntry => { + const obj = objectsByKey.get(key); + return { + key, + lastModified: obj?.LastModified?.toISOString(), + size: obj?.Size, + }; +}; + +export function buildRecordsManifest(objects: _Object[], generated: string): RecordsManifestV2 { + const objectsByKey = new Map(); + for (const obj of objects) { + if (obj.Key !== undefined) { + objectsByKey.set(obj.Key, obj); + } + } + + for (const key of REQUIRED_SUMMARY_KEYS) { + if (!objectsByKey.has(key)) { + console.warn(`Missing summary key in bucket listing: ${key}`); + } + } + if (!objectsByKey.has(PLAYER_SUMMARY_MANIFEST_KEY)) { + console.warn(`Missing player summary manifest in bucket listing: ${PLAYER_SUMMARY_MANIFEST_KEY}`); + } + + return { + version: 2, + generated, + summaryFiles: { + monolith: summaryEntry(objectsByKey, SUMMARY_MONOLITH_KEY), + site: summaryEntry(objectsByKey, SUMMARY_SITE_KEY), + players: summaryEntry(objectsByKey, SUMMARY_PLAYERS_KEY), + ratings: summaryEntry(objectsByKey, SUMMARY_RATINGS_KEY), + playerSummaryPattern: PLAYER_SUMMARY_KEY_PATTERN, + playerManifest: summaryEntry(objectsByKey, PLAYER_SUMMARY_MANIFEST_KEY), + }, + objects, + }; +} diff --git a/crons/src/utils/resolveGameVariants.test.ts b/crons/src/utils/resolveGameVariants.test.ts new file mode 100644 index 00000000..cdeed915 --- /dev/null +++ b/crons/src/utils/resolveGameVariants.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from "vitest"; +import { + reconcileVariantsInGameState, + resolveGameVariantUids, +} from "./resolveGameVariants.js"; + +describe("resolveGameVariantUids", () => { + it("returns state variants when both match", () => { + expect(resolveGameVariantUids(["scrambled"], ["scrambled"])).toEqual(["scrambled"]); + }); + + it("returns record variants when state is empty (retroactive bug)", () => { + expect(resolveGameVariantUids([], ["scrambled"])).toEqual(["scrambled"]); + expect(resolveGameVariantUids(undefined, ["scrambled"])).toEqual(["scrambled"]); + }); + + it("returns state variants when record is empty", () => { + expect(resolveGameVariantUids(["8x10"], [])).toEqual(["8x10"]); + expect(resolveGameVariantUids(["8x10"], undefined)).toEqual(["8x10"]); + }); + + it("returns empty when both are empty", () => { + expect(resolveGameVariantUids([], [])).toEqual([]); + expect(resolveGameVariantUids(undefined, undefined)).toEqual([]); + }); + + it("prefers record and warns when both non-empty and different", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(resolveGameVariantUids(["cross"], ["scrambled"], { + metaGame: "amazons", + gameId: "abc", + })).toEqual(["scrambled"]); + expect(warn).toHaveBeenCalledOnce(); + warn.mockRestore(); + }); + + it("canonicalizes order for comparison", () => { + expect(resolveGameVariantUids(["b", "a"], ["a", "b"])).toEqual(["a", "b"]); + }); +}); + +describe("reconcileVariantsInGameState", () => { + it("patches empty state variants from record", () => { + const state = JSON.stringify({ game: "amazons", variants: [], stack: [] }); + const patched = reconcileVariantsInGameState(state, ["scrambled"]); + expect(JSON.parse(patched).variants).toEqual(["scrambled"]); + }); + + it("returns original JSON when no record variants", () => { + const state = JSON.stringify({ game: "amazons", variants: [], stack: [] }); + expect(reconcileVariantsInGameState(state, [])).toBe(state); + expect(reconcileVariantsInGameState(state, undefined)).toBe(state); + }); + + it("returns original JSON when state already matches", () => { + const state = JSON.stringify({ game: "amazons", variants: ["scrambled"], stack: [] }); + expect(reconcileVariantsInGameState(state, ["scrambled"])).toBe(state); + }); + + it("returns original JSON when state is invalid", () => { + expect(reconcileVariantsInGameState("not-json", ["scrambled"])).toBe("not-json"); + }); +}); diff --git a/crons/src/utils/resolveGameVariants.ts b/crons/src/utils/resolveGameVariants.ts new file mode 100644 index 00000000..44d16e6f --- /dev/null +++ b/crons/src/utils/resolveGameVariants.ts @@ -0,0 +1,66 @@ +import { variantComboKey } from "./recordGameId.js"; + +export type ResolveGameVariantsContext = { + metaGame?: string; + gameId?: string; +}; + +function normalizeVariants(variants: string[] | undefined): string[] { + if (variants === undefined || variants.length === 0) { + return []; + } + return [...new Set(variants)].sort(); +} + +function variantsEqual( + a: readonly string[], + b: readonly string[], +): boolean { + return variantComboKey(a) === variantComboKey(b); +} + +/** Prefer record variants when serialized state lost them (historical gameslib bug). */ +export function resolveGameVariantUids( + stateVariants: string[] | undefined, + recordVariants: string[] | undefined, + context?: ResolveGameVariantsContext, +): string[] { + const state = normalizeVariants(stateVariants); + const record = normalizeVariants(recordVariants); + + if (variantsEqual(state, record)) { + return state; + } + if (state.length === 0 && record.length > 0) { + return record; + } + if (record.length === 0 && state.length > 0) { + return state; + } + console.warn( + `Variant mismatch for ${context?.metaGame ?? "unknown"} game ${context?.gameId ?? "unknown"}: ` + + `state=${JSON.stringify(state)} record=${JSON.stringify(record)}; preferring record`, + ); + return record; +} + +export function reconcileVariantsInGameState( + stateJson: string, + recordVariants: string[] | undefined, +): string { + if (recordVariants === undefined || recordVariants.length === 0) { + return stateJson; + } + let parsed: { variants?: string[] }; + try { + parsed = JSON.parse(stateJson) as { variants?: string[] }; + } catch { + return stateJson; + } + const resolved = resolveGameVariantUids(parsed.variants, recordVariants); + if (variantsEqual(parsed.variants ?? [], resolved)) { + return stateJson; + } + parsed.variants = resolved; + return JSON.stringify(parsed); +} diff --git a/crons/src/utils/resolveRenderLabels.test.ts b/crons/src/utils/resolveRenderLabels.test.ts new file mode 100644 index 00000000..69e522b8 --- /dev/null +++ b/crons/src/utils/resolveRenderLabels.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, beforeAll } from "vitest"; +import { addResource } from "@abstractplay/gameslib"; +import type { APRenderRep } from "@abstractplay/renderer"; +import enApgames from "@abstractplay/gameslib/locales/en/apgames.json"; +import enApresults from "@abstractplay/gameslib/locales/en/apresults.json"; +import { resolveRenderLabels } from "./resolveRenderLabels.js"; + +describe("resolveRenderLabels", () => { + const players = [ + { name: "Alice" }, + { name: "Bob" }, + ]; + + const t = (key: string, params?: Record) => { + if (key === "test:STASH") { + return `${params?.player}'s stash`; + } + return key; + }; + + it("resolves structured area labels to display names", () => { + const rep = { + areas: [ + { + type: "pieces", + label: { + textKey: "test:STASH", + actor: { kind: "seat", seat: 2 }, + }, + pieces: ["A1"], + }, + ], + legend: { A1: "piece" }, + } as unknown as APRenderRep; + const resolved = resolveRenderLabels(rep, players, t); + expect((resolved.areas![0] as { label: string }).label).toBe("Bob's stash"); + }); + + it("leaves plain-string labels unchanged", () => { + const rep = { + areas: [ + { + type: "pieces", + label: "Player 1 hand", + pieces: ["A1"], + }, + ], + } as unknown as APRenderRep; + const resolved = resolveRenderLabels(rep, players, t); + expect((resolved.areas![0] as { label: string }).label).toBe("Player 1 hand"); + }); + + it("resolves streetcar-style taken area labels", () => { + const rep = { + areas: [ + { + type: "pieces", + label: { + textKey: "apgames:validation.streetcar.TAKEN_LABEL", + actor: { kind: "seat", seat: 1 }, + }, + pieces: ["E"], + }, + ], + } as unknown as APRenderRep; + const streetcarT = (key: string, params?: Record) => { + if (key === "apgames:validation.streetcar.TAKEN_LABEL") { + return `${params?.player}'s housing limits`; + } + return key; + }; + const resolved = resolveRenderLabels(rep, players, streetcarT); + expect((resolved.areas![0] as { label: string }).label).toBe("Alice's housing limits"); + }); + + it("resolves entropy board labels and board markers", () => { + const rep = { + board: { + style: "squares", + boardOne: { + label: { + textKey: "test:STASH", + actor: { kind: "seat", seat: 1 }, + }, + }, + markers: [ + { + type: "label", + label: { + textKey: "test:STASH", + actor: { kind: "seat", seat: 2 }, + }, + points: [ + { row: 0, col: 0 }, + { row: 0, col: 1 }, + ], + }, + ], + }, + } as unknown as APRenderRep; + const resolved = resolveRenderLabels(rep, players, t); + const board = resolved.board as { + boardOne: { label: string }; + markers: { label: string }[]; + }; + expect(board.boardOne.label).toBe("Alice's stash"); + expect(board.markers[0].label).toBe("Bob's stash"); + }); + + it("resolves structured labels in every multiframe render rep", () => { + const stashLabel = { + textKey: "test:STASH", + actor: { kind: "seat", seat: 1 }, + }; + const frame = { + areas: [ + { + type: "pieces", + label: stashLabel, + pieces: ["A1"], + }, + ], + } as unknown as APRenderRep; + const rep = [structuredClone(frame), structuredClone(frame)] as APRenderRep[]; + const resolved = resolveRenderLabels(rep, players, t); + expect(Array.isArray(resolved)).toBe(true); + for (const item of resolved as APRenderRep[]) { + expect((item.areas![0] as { label: string }).label).toBe("Alice's stash"); + } + }); + + describe("with real apgames bundle", () => { + beforeAll(async () => { + addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + }); + + it("resolves streetcar TAKEN_LABEL via i18next", () => { + const gamesI18n = addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + const rep = { + areas: [ + { + type: "pieces", + label: { + textKey: "apgames:validation.streetcar.TAKEN_LABEL", + actor: { kind: "seat", seat: 1 }, + }, + pieces: ["E"], + }, + ], + } as unknown as APRenderRep; + const resolved = resolveRenderLabels(rep, players, (key, params) => + String(gamesI18n.t(key, params ?? {})), + ); + expect((resolved.areas![0] as { label: string }).label).toBe("Alice's housing limits"); + }); + + it("resolves rincala LABEL_STASH on multiframe reps via i18next", () => { + const gamesI18n = addResource("en", undefined, { + bundles: { apgames: enApgames, apresults: enApresults }, + }); + const stashLabel = { + textKey: "apgames:validation.rincala.LABEL_STASH", + actor: { kind: "seat", seat: 2 }, + }; + const frame = { + areas: [ + { + type: "pieces", + label: stashLabel, + pieces: ["Y"], + }, + ], + } as unknown as APRenderRep; + const resolved = resolveRenderLabels([frame, structuredClone(frame)], players, (key, params) => + String(gamesI18n.t(key, params ?? {})), + ); + const frames = resolved as APRenderRep[]; + expect(frames).toHaveLength(2); + const label = (frames[1].areas![0] as { label: string }).label; + expect(label).not.toContain("apgames:"); + expect(label).not.toBe("apgames:validation.rincala.LABEL_STASH"); + }); + }); +}); diff --git a/crons/src/utils/resolveRenderLabels.ts b/crons/src/utils/resolveRenderLabels.ts new file mode 100644 index 00000000..44e75c80 --- /dev/null +++ b/crons/src/utils/resolveRenderLabels.ts @@ -0,0 +1,116 @@ +import { + isStructuredRenderLabel, + resolveRenderLabel, + type ChatLogTranslate, + type RenderLabel, +} from "@abstractplay/gameslib"; +import type { APRenderRep } from "@abstractplay/renderer"; +import type { ThumbnailRenderOutput } from "./thumbnailRenderRep.js"; + +type ThumbnailPlayer = { + name: string; +}; + +type LabelHost = { + label?: RenderLabel; + type?: string; + buttons?: { label?: RenderLabel }[]; +}; + +type BoardHost = { + boardOne?: { label?: RenderLabel }; + boardTwo?: { label?: RenderLabel }; + markers?: { type?: string; label?: RenderLabel }[]; +}; + +function resolveLabelField( + label: RenderLabel, + playerNames: string[], + t: ChatLogTranslate, +): string | RenderLabel { + if (!isStructuredRenderLabel(label)) { + return label; + } + return resolveRenderLabel(label, playerNames, t); +} + +function walkMarkers( + markers: BoardHost["markers"], + playerNames: string[], + t: ChatLogTranslate, +): void { + if (!Array.isArray(markers)) { + return; + } + for (const marker of markers) { + if (marker?.type === "label" && marker.label !== undefined) { + marker.label = resolveLabelField(marker.label, playerNames, t); + } + } +} + +function walkAreas( + areas: LabelHost[] | undefined, + playerNames: string[], + t: ChatLogTranslate, +): void { + if (!Array.isArray(areas)) { + return; + } + for (const area of areas) { + if (area?.label !== undefined) { + area.label = resolveLabelField(area.label, playerNames, t); + } + if (area?.type === "buttonBar" && Array.isArray(area.buttons)) { + for (const button of area.buttons) { + if (button?.label !== undefined) { + button.label = resolveLabelField(button.label, playerNames, t); + } + } + } + } +} + +function walkBoard( + board: BoardHost | null | undefined, + playerNames: string[], + t: ChatLogTranslate, +): void { + if (!board || typeof board !== "object") { + return; + } + if (board.boardOne?.label !== undefined) { + board.boardOne.label = resolveLabelField(board.boardOne.label, playerNames, t); + } + if (board.boardTwo?.label !== undefined) { + board.boardTwo.label = resolveLabelField(board.boardTwo.label, playerNames, t); + } + walkMarkers(board.markers, playerNames, t); +} + +function resolveRenderLabelsOne( + rep: APRenderRep, + playerNames: string[], + t: ChatLogTranslate, +): APRenderRep { + if (!rep || typeof rep !== "object" || Array.isArray(rep)) { + return rep; + } + const out = structuredClone(rep); + walkBoard(out.board as BoardHost | null | undefined, playerNames, t); + walkAreas(out.areas as LabelHost[] | undefined, playerNames, t); + return out; +} + +/** Resolve structured render labels to display strings before thumbnail SVG rendering. */ +export function resolveRenderLabels( + rep: ThumbnailRenderOutput, + players: ThumbnailPlayer[], + t: ChatLogTranslate, +): ThumbnailRenderOutput { + const playerNames = players.map((p) => p.name); + if (Array.isArray(rep)) { + return rep.map((frame) => resolveRenderLabelsOne(frame, playerNames, t)); + } + return resolveRenderLabelsOne(rep, playerNames, t); +} diff --git a/crons/src/utils/streamJsonArray.test.ts b/crons/src/utils/streamJsonArray.test.ts new file mode 100644 index 00000000..27d3bf96 --- /dev/null +++ b/crons/src/utils/streamJsonArray.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { Readable } from "node:stream"; +import { streamJsonArrayFromReadable } from "./streamJsonArray.js"; + +describe("streamJsonArray", () => { + it("yields each array element without JSON.parse on the full buffer", async () => { + const items = [{ id: 1 }, { id: 2 }, { id: 3 }]; + const readable = Readable.from([JSON.stringify(items)]); + const seen: number[] = []; + const count = await streamJsonArrayFromReadable<{ id: number }>(readable, (item) => { + seen.push(item.id); + }); + expect(count).toBe(3); + expect(seen).toEqual([1, 2, 3]); + }); +}); diff --git a/crons/src/utils/streamJsonArray.ts b/crons/src/utils/streamJsonArray.ts new file mode 100644 index 00000000..7ca8adce --- /dev/null +++ b/crons/src/utils/streamJsonArray.ts @@ -0,0 +1,45 @@ +import { createRequire } from "node:module"; +import { GetObjectCommand, type S3Client } from "@aws-sdk/client-s3"; +import type { Readable, Transform } from "node:stream"; + +const require = createRequire(import.meta.url); + +/** createRequire — stream-json is CJS; ESM named imports fail on Lambda. */ +const { parser } = require("stream-json") as { parser: () => Transform }; +const { streamArray } = require("stream-json/streamers/StreamArray") as { + streamArray: () => Transform; +}; + +/** + * Stream-parse a top-level JSON array without holding the full file string in memory. + */ +export async function streamJsonArrayFromReadable( + readable: Readable, + onItem: (item: T) => void, +): Promise { + const jsonParser = parser(); + const arrayStreamer = streamArray(); + readable.pipe(jsonParser).pipe(arrayStreamer); + + let count = 0; + for await (const chunk of arrayStreamer) { + const row = chunk as unknown as { value: T }; + onItem(row.value); + count++; + } + return count; +} + +export async function streamJsonArrayFromS3( + s3: S3Client, + bucket: string, + key: string, + onItem: (item: T) => void, +): Promise { + const response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + const body = response.Body; + if (body === undefined) { + throw new Error(`Unable to load s3://${bucket}/${key}`); + } + return streamJsonArrayFromReadable(body as Readable, onItem); +} diff --git a/crons/src/utils/summaryRatings.ts b/crons/src/utils/summaryRatings.ts new file mode 100644 index 00000000..fc6ef746 --- /dev/null +++ b/crons/src/utils/summaryRatings.ts @@ -0,0 +1,49 @@ +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { REC_BUCKET, SUMMARY_RATINGS_KEY } from "../constants/recordsBucket.js"; +import type { UserGameRating } from "types/stats/UserGameRating.js"; + +const s3 = new S3Client({ region: "us-east-1" }); + +let cachedHighest: UserGameRating[] | undefined; +let cacheLoaded = false; + +type SummaryRatingsTier = { + tier?: string; + generated?: string; + ratings: { + highest: UserGameRating[]; + }; +}; + +export async function loadSummaryRatingsHighest(): Promise { + if (cacheLoaded && cachedHighest !== undefined) { + return cachedHighest; + } + const response = await s3.send(new GetObjectCommand({ + Bucket: REC_BUCKET, + Key: SUMMARY_RATINGS_KEY, + })); + const body = await response.Body?.transformToString(); + if (body === undefined) { + throw new Error(`Unable to load s3://${REC_BUCKET}/${SUMMARY_RATINGS_KEY}`); + } + const parsed = JSON.parse(body) as SummaryRatingsTier; + const highest = parsed.ratings?.highest; + if (highest === undefined) { + throw new Error(`Missing ratings.highest in ${SUMMARY_RATINGS_KEY}`); + } + cachedHighest = highest; + cacheLoaded = true; + return highest; +} + +/** @internal test helper */ +export function clearSummaryRatingsCacheForTests(): void { + cachedHighest = undefined; + cacheLoaded = false; +} + +export function setSummaryRatingsHighestForTests(highest: UserGameRating[]): void { + cachedHighest = highest; + cacheLoaded = true; +} diff --git a/crons/src/utils/thumbnailConfig.ts b/crons/src/utils/thumbnailConfig.ts new file mode 100644 index 00000000..e3b23fa7 --- /dev/null +++ b/crons/src/utils/thumbnailConfig.ts @@ -0,0 +1,5 @@ +/** Public thumbnail CDN bucket (JSON + SVG). */ +export const THUMB_BUCKET = "thumbnails.abstractplay.com"; + +/** Metas skipped for prerender/SVG (JSON-only or known broken). */ +export const THUMBNAIL_BROKEN_METAS: readonly string[] = []; diff --git a/crons/src/utils/thumbnailFreshness.test.ts b/crons/src/utils/thumbnailFreshness.test.ts new file mode 100644 index 00000000..025b8841 --- /dev/null +++ b/crons/src/utils/thumbnailFreshness.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { findThumbnailFreshnessMismatches } from "./thumbnailFreshness.js"; + +describe("findThumbnailFreshnessMismatches", () => { + const jsonDate = new Date("2026-09-09T06:05:00.000Z"); + const olderSvg = new Date("2026-09-08T06:04:00.000Z"); + const newerSvg = new Date("2026-09-09T06:05:21.000Z"); + + it("flags svg older than json", () => { + const heads = new Map([ + ["rincala.json", { key: "rincala.json", lastModified: jsonDate }], + ["rincala-light.svg", { key: "rincala-light.svg", lastModified: olderSvg }], + ]); + const mismatches = findThumbnailFreshnessMismatches(["rincala"], heads); + expect(mismatches).toHaveLength(1); + expect(mismatches[0]).toMatchObject({ + meta: "rincala", + reason: "svg-older-than-json", + }); + }); + + it("passes when svg is newer than json", () => { + const heads = new Map([ + ["frogger.json", { key: "frogger.json", lastModified: jsonDate }], + ["frogger-light.svg", { key: "frogger-light.svg", lastModified: newerSvg }], + ]); + expect(findThumbnailFreshnessMismatches(["frogger"], heads)).toHaveLength(0); + }); + + it("flags missing svg", () => { + const heads = new Map([ + ["rincala.json", { key: "rincala.json", lastModified: jsonDate }], + ]); + const mismatches = findThumbnailFreshnessMismatches(["rincala"], heads); + expect(mismatches[0].reason).toBe("missing-svg"); + }); + + it("skips broken metas", () => { + const heads = new Map([ + ["broken.json", { key: "broken.json", lastModified: jsonDate }], + ]); + expect(findThumbnailFreshnessMismatches(["broken"], heads, ["broken"])).toHaveLength(0); + }); + + it("ignores metas with no json object", () => { + expect(findThumbnailFreshnessMismatches(["missing"], new Map())).toHaveLength(0); + }); +}); diff --git a/crons/src/utils/thumbnailFreshness.ts b/crons/src/utils/thumbnailFreshness.ts new file mode 100644 index 00000000..933d4475 --- /dev/null +++ b/crons/src/utils/thumbnailFreshness.ts @@ -0,0 +1,62 @@ +export type ObjectHead = { + key: string; + lastModified: Date; +}; + +export type ThumbnailFreshnessMismatch = { + meta: string; + jsonKey: string; + svgKey: string; + jsonLastModified: string; + svgLastModified: string | null; + reason: "missing-svg" | "svg-older-than-json"; +}; + +/** + * Metas whose light SVG is missing or older than the JSON thumbnail. + * `brokenMetas` are excluded (JSON-only games). + */ +export function findThumbnailFreshnessMismatches( + metas: string[], + heads: Map, + brokenMetas: readonly string[] = [], +): ThumbnailFreshnessMismatch[] { + const broken = new Set(brokenMetas); + const mismatches: ThumbnailFreshnessMismatch[] = []; + + for (const meta of metas) { + if (broken.has(meta)) { + continue; + } + const jsonKey = `${meta}.json`; + const svgKey = `${meta}-light.svg`; + const jsonHead = heads.get(jsonKey); + if (!jsonHead) { + continue; + } + const svgHead = heads.get(svgKey); + if (!svgHead) { + mismatches.push({ + meta, + jsonKey, + svgKey, + jsonLastModified: jsonHead.lastModified.toISOString(), + svgLastModified: null, + reason: "missing-svg", + }); + continue; + } + if (svgHead.lastModified < jsonHead.lastModified) { + mismatches.push({ + meta, + jsonKey, + svgKey, + jsonLastModified: jsonHead.lastModified.toISOString(), + svgLastModified: svgHead.lastModified.toISOString(), + reason: "svg-older-than-json", + }); + } + } + + return mismatches.sort((a, b) => a.meta.localeCompare(b.meta)); +} diff --git a/crons/src/utils/thumbnailRenderRep.test.ts b/crons/src/utils/thumbnailRenderRep.test.ts new file mode 100644 index 00000000..eca84bd0 --- /dev/null +++ b/crons/src/utils/thumbnailRenderRep.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import type { APRenderRep } from "@abstractplay/renderer"; +import { coalesceRenderFrames } from "./thumbnailRenderRep.js"; + +describe("coalesceRenderFrames", () => { + const frameA = { renderer: "stacking-offset", pieces: "A" } as unknown as APRenderRep; + const frameB = { renderer: "stacking-offset", pieces: "B" } as unknown as APRenderRep; + + it("returns a single rep unchanged", () => { + expect(coalesceRenderFrames(frameA)).toBe(frameA); + }); + + it("returns the last frame from a multiframe array", () => { + expect(coalesceRenderFrames([frameA, frameB])).toBe(frameB); + }); + + it("throws when no drawable frame exists", () => { + expect(() => coalesceRenderFrames([])).toThrow(/no drawable frame/); + }); +}); diff --git a/crons/src/utils/thumbnailRenderRep.ts b/crons/src/utils/thumbnailRenderRep.ts new file mode 100644 index 00000000..d829640c --- /dev/null +++ b/crons/src/utils/thumbnailRenderRep.ts @@ -0,0 +1,13 @@ +import type { APRenderRep } from "@abstractplay/renderer"; + +/** Output of `game.render()` — single frame or animation frames. */ +export type ThumbnailRenderOutput = APRenderRep | APRenderRep[]; + +/** Pick the last frame for static thumbnail SVG rendering. */ +export function coalesceRenderFrames(rep: ThumbnailRenderOutput): APRenderRep { + const frame = Array.isArray(rep) ? rep.at(-1) : rep; + if (!frame || typeof frame !== "object" || Array.isArray(frame)) { + throw new Error("Thumbnail render rep has no drawable frame"); + } + return frame; +} diff --git a/crons/test/fixtures/batch-ratings.json b/crons/test/fixtures/batch-ratings.json new file mode 100644 index 00000000..196ba64b --- /dev/null +++ b/crons/test/fixtures/batch-ratings.json @@ -0,0 +1,68 @@ +{ + "highest": [ + { + "user": "alice", + "game": "chess (no variants)", + "rating": 1350, + "wld": [10, 5, 2], + "glicko": { + "rating": 1320, + "rd": 60, + "volatility": 0.06, + "ratingLow": 1200, + "ratingHigh": 1440, + "provisional": false, + "established": true, + "n": 17 + } + }, + { + "user": "bob", + "game": "chess (no variants)", + "rating": 1280, + "wld": [8, 8, 1], + "glicko": { + "rating": 1250, + "rd": 80, + "volatility": 0.06, + "ratingLow": 1090, + "ratingHigh": 1410, + "provisional": false, + "established": false, + "n": 17 + } + }, + { + "user": "alice", + "game": "go (#board|#ruleset)", + "rating": 1400, + "wld": [3, 1, 0], + "glicko": { + "rating": 1380, + "rd": 90, + "volatility": 0.06, + "ratingLow": 1200, + "ratingHigh": 1560, + "provisional": true, + "established": false, + "n": 4 + } + }, + { + "user": "carol", + "game": "go (#board|#ruleset)", + "rating": 1300, + "wld": [2, 2, 0], + "glicko": { + "rating": 1310, + "rd": 70, + "volatility": 0.06, + "ratingLow": 1170, + "ratingHigh": 1450, + "provisional": false, + "established": false, + "n": 4 + } + } + ] +} diff --git a/crons/tsconfig.json b/crons/tsconfig.json new file mode 100644 index 00000000..70bb0387 --- /dev/null +++ b/crons/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "noEmit": true, + "baseUrl": "./src" + }, + "include": [ + "src/**/*.ts", + "scripts/**/*.ts", + "bin/**/*.ts", + "vitest.config.ts" + ] +} diff --git a/crons/vitest.config.ts b/crons/vitest.config.ts new file mode 100644 index 00000000..c12de096 --- /dev/null +++ b/crons/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + }, +}); diff --git a/docs/deployment.md b/docs/deployment.md index 8752a531..6d6cd4bb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -13,11 +13,20 @@ GitHub Actions deploy via Serverless Framework: Downstream repos (e.g. gameslib) can trigger backend redeploys after package publishes. +Each deploy runs **two** Serverless stacks (API first, then crons): + +| Stack | Service name | Deploy (CI) | +|-------|----------------|-------------| +| API / WebSocket | `abstract-play` | `bash bin/serverless-deploy.sh ` | +| Scheduled jobs | `abstract-play-backend-crons` | `bash crons/bin/serverless-deploy.sh ` | + +Crons source lives in [`crons/`](../crons/). See [Crons deployment](/crons/deployment/). + ## AP dependency pins (`ci-deps.*.json`) Canonical pins live in `ci-deps.dev.json` and `ci-deps.prod.json`. CI runs `npm ci` → manifest validation → `ap-install-deps --stage dev|prod` → strict lockfile check → build/test. -After a merge that touches dependency files, run `npm run sync-deps` on `develop` (or `npm run sync-deps:prod` on `main`) and commit `ci-deps.*.json`, `package.json`, and `package-lock.json` together. Do not hand-merge AP version strings in `package.json`. +After a merge that touches dependency files, run `npm run sync-deps` on `develop` (or `npm run sync-deps:prod` on `main`) and commit root and `crons/` `ci-deps.*.json`, `package.json`, and the root `package-lock.json` together. `npm run sync-deps` runs `ap-install-deps` at the repo root and updates `crons/package.json` / `crons/ci-deps.*.json` from the lockfile. Do not hand-merge AP version strings in `package.json`. `ci-deps.prod.json` is protected on `main` via `.gitattributes` (`merge=ours`). `ci-deps.dev.json` is protected on `develop` the same way (e.g. when merging `l10n/weblate`). `package.json` and `package-lock.json` are regenerated via `sync-deps`, not merge=ours. @@ -130,7 +139,7 @@ Bot pools are separate per stage — see [Bots](/backend/subsystems/bots/). ## Documentation deploys -When a push to `develop` or `main` includes changes under `docs/`, the deploy workflow dispatches `dep_update_dev` / `dep_update_prod` to the [docs](https://github.com/AbstractPlay/docs) repository so the site rebuilds with updated submodule content. +When a push to `develop` or `main` includes changes under `docs/` or `crons/docs/`, the deploy workflow dispatches `dep_update_dev` / `dep_update_prod` to the [docs](https://github.com/AbstractPlay/docs) repository so the site rebuilds (after the docs repo vendors this monorepo and syncs `/crons/` — see `crons/docs/_docs-repo-integration.md`). ## Related diff --git a/package-lock.json b/package-lock.json index d9a299ed..3830ca6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,53 +8,791 @@ "name": "abstractplay-backend", "version": "1.0.0-beta", "license": "MIT", + "workspaces": [ + "crons" + ], + "dependencies": { + "@abstractplay/gameslib": "1.0.0-ci-35453221145.0", + "@abstractplay/recranks": "1.0.0-ci-35280155165.0", + "@abstractplay/renderer": "1.0.0-ci-35278756813.0", + "@aws-sdk/client-apigatewaymanagementapi": "3.1090.0", + "@aws-sdk/client-cognito-identity-provider": "3.1090.0", + "@aws-sdk/client-dynamodb": "3.1090.0", + "@aws-sdk/client-s3": "3.1090.0", + "@aws-sdk/client-ses": "3.1090.0", + "@aws-sdk/client-sqs": "3.1090.0", + "@aws-sdk/lib-dynamodb": "3.1090.0", + "@aws-sdk/s3-request-presigner": "3.1090.0", + "@sunknudsen/totp": "^1.1.0", + "aws-jwt-verify": "^5.1.1", + "fast-xml-parser": "^5.11.1", + "fflate": "^0.8.1", + "i18next": "^22.4.15", + "ion-js": "^5.2.0", + "lodash": "^4.17.21", + "uuid": "^11.1.1", + "web-push": "^3.6.3" + }, + "devDependencies": { + "@abstractplay/ap-deps-tools": "^1.1.1", + "@aws-sdk/types": "^3.310.0", + "@google/genai": "^2.13.0", + "@types/aws-lambda": "^8.10.115", + "@types/node": "^24", + "@types/uuid": "^9.0.1", + "@types/web-push": "^3.3.2", + "@typescript-eslint/eslint-plugin": "^5.59.1", + "@typescript-eslint/parser": "^5.59.1", + "aws-amplify": "6.20.0", + "eslint": "^8.39.0", + "fs-extra": "^11.3.4", + "i": "^0.3.7", + "serverless": "4.42.0", + "serverless-esbuild": "^1.57.0", + "serverless-plugin-common-excludes": "^4.0.0", + "serverless-scriptable-plugin": "^1.3.1", + "tsx": "^4.8.1", + "typescript": "^5.0.4", + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=24", + "npm": "11.6.2" + } + }, + "crons": { + "name": "abstractplay-backend-crons", + "version": "1.0.0-beta", + "license": "MIT", + "dependencies": { + "@abstractplay/gameslib": "1.0.0-ci-35453221145.0", + "@abstractplay/recranks": "1.0.0-ci-35280155165.0", + "@abstractplay/renderer": "1.0.0-ci-35278756813.0", + "@aws-sdk/client-cloudwatch": "^3.1128.0", + "@aws-sdk/client-dynamodb": "^3.321.1", + "@aws-sdk/client-s3": "^3.374.0", + "@aws-sdk/client-ses": "^3.321.1", + "@aws-sdk/client-sqs": "^3.374.0", + "@aws-sdk/lib-dynamodb": "^3.321.1", + "@sparticuz/chromium": "^143.0.0", + "aws-lambda": "^1.0.7", + "fflate": "^0.8.1", + "i18next": "^22.4.15", + "ion-js": "^5.2.0", + "nanoid": "^5.1.5", + "puppeteer-core": "^24.33.0", + "stream-json": "^1.8.0", + "uuid": "^11.1.1", + "web-push": "^3.6.3" + }, + "devDependencies": { + "@abstractplay/ap-deps-tools": "^1.1.1", + "@aws-sdk/client-cloudfront": "^3.374.0", + "@aws-sdk/types": "^3.310.0", + "@types/aws-lambda": "^8.10.115", + "@types/node": "^20.19.0", + "@types/web-push": "^3.3.2", + "@typescript-eslint/eslint-plugin": "^5.59.1", + "@typescript-eslint/parser": "^5.59.1", + "esbuild": "^0.27.1", + "eslint": "^8.39.0", + "fs-extra": "^11.3.4", + "rimraf": "^6.1.2", + "serverless": "4.42.0", + "serverless-esbuild": "^1.56.1", + "serverless-scriptable-plugin": "^1.3.1", + "tsx": "^4.20.5", + "typescript": "^5.0.4", + "vite": "^6.4.1", + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=24", + "npm": "11.6.2" + } + }, + "crons/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "crons/node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "crons/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "crons/node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "crons/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "crons/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "license": "BlueOak-1.0.0", "dependencies": { - "@abstractplay/gameslib": "1.0.0-ci-35279226489.0", - "@abstractplay/recranks": "latest", - "@abstractplay/renderer": "1.0.0-ci-35278756813.0", - "@aws-sdk/client-apigatewaymanagementapi": "3.1090.0", - "@aws-sdk/client-cognito-identity-provider": "3.1090.0", - "@aws-sdk/client-dynamodb": "3.1090.0", - "@aws-sdk/client-s3": "3.1090.0", - "@aws-sdk/client-ses": "3.1090.0", - "@aws-sdk/client-sqs": "3.1090.0", - "@aws-sdk/lib-dynamodb": "3.1090.0", - "@aws-sdk/s3-request-presigner": "3.1090.0", - "@sunknudsen/totp": "^1.1.0", - "aws-jwt-verify": "^5.1.1", - "fast-xml-parser": "^5.11.1", - "fflate": "^0.8.1", - "i18next": "^22.4.15", - "ion-js": "^5.2.0", - "lodash": "^4.17.21", - "uuid": "^11.1.1", - "web-push": "^3.6.3" + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" }, - "devDependencies": { - "@abstractplay/ap-deps-tools": "^1.1.1", - "@aws-sdk/types": "^3.310.0", - "@google/genai": "^2.13.0", - "@types/aws-lambda": "^8.10.115", - "@types/node": "^24", - "@types/uuid": "^9.0.1", - "@types/web-push": "^3.3.2", - "@typescript-eslint/eslint-plugin": "^5.59.1", - "@typescript-eslint/parser": "^5.59.1", - "aws-amplify": "6.20.0", - "eslint": "^8.39.0", - "fs-extra": "^11.3.4", - "i": "^0.3.7", - "serverless": "4.42.0", - "serverless-esbuild": "^1.57.0", - "serverless-plugin-common-excludes": "^4.0.0", - "serverless-scriptable-plugin": "^1.3.1", + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "crons/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "crons/node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "crons/node_modules/rimraf": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.1.3.tgz", + "integrity": "sha512-LKg+Cr2ZF61fkcaK1UdkH2yEBBKnYjTyWzTJT6KNPcSPaiT7HSdhtMXQuN5wkTX0Xu72KQ1l8S42rlmexS2hSA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "glob": "^13.0.3", + "package-json-from-dist": "^1.0.1" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "crons/node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "crons/node_modules/vite": { + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", + "postcss": "^8.5.3", + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", "tsx": "^4.8.1", - "typescript": "^5.0.4", - "vitest": "^3.2.4" + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "crons/node_modules/vite/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" }, "engines": { - "node": ">=24", - "npm": "11.6.2" + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" } }, "node_modules/@abstractplay/ap-deps-tools": { @@ -73,12 +811,12 @@ } }, "node_modules/@abstractplay/gameslib": { - "version": "1.0.0-ci-35279226489.0", - "resolved": "https://npm.pkg.github.com/download/@abstractplay/gameslib/1.0.0-ci-35279226489.0/e44faea4fe214e3205c920e4f8bd39ad5adb26e2", - "integrity": "sha512-uKm1rdkSNREm2sKXPOhvFTQWDZH0tAo1Z3xVcRV9Hi/gmkq9gLjpbaGS1U58a4lyeu45zrhJ2sO98R/re1uNaQ==", + "version": "1.0.0-ci-35453221145.0", + "resolved": "https://npm.pkg.github.com/download/@abstractplay/gameslib/1.0.0-ci-35453221145.0/668a05509f916463a2547db73bc7aea9dc254543", + "integrity": "sha512-kuH7+nQvN0rP1USWUG8sa/wVgKEgWUKqjE+tSndHE09w5nbSdtqWcj/mCaZSCVCvdGXAMVZXjeOAfVG8YlPbNg==", "license": "MIT", "dependencies": { - "@abstractplay/recranks": "1.0.0-ci-35004500052.0", + "@abstractplay/recranks": "1.0.0-ci-35280155165.0", "@abstractplay/renderer": "1.0.0-ci-35278756813.0", "@turf/boolean-contains": "^6.5.0", "@turf/boolean-intersects": "^6.5.0", @@ -133,9 +871,9 @@ } }, "node_modules/@abstractplay/recranks": { - "version": "1.0.0-ci-35004500052.0", - "resolved": "https://npm.pkg.github.com/download/@abstractplay/recranks/1.0.0-ci-35004500052.0/1f9ca68065545404120ea477c517ed2f6efef46e", - "integrity": "sha512-yYG3RUJcpKyxniTELiHM3kLawT0GswvKjmUwqtzG8X9rFHYMd/XNKx+iDzPOydRE/VPKcSRyiJvgE7hPzmhoJg==", + "version": "1.0.0-ci-35280155165.0", + "resolved": "https://npm.pkg.github.com/download/@abstractplay/recranks/1.0.0-ci-35280155165.0/f006effbc3973ae8f10dfa99ea76297550d0b0a8", + "integrity": "sha512-ppbn9wh2Xy1scx0ZdtarBfbxat6x7hQxHwbL7uUrT7ijC9YPvbyuHh8RScgmSEVSEr3lhYkkzxsXseHzTS8jKQ==", "license": "MIT", "dependencies": { "ajv": "^8.12.0", @@ -360,7 +1098,6 @@ "integrity": "sha512-u1nuT1YAdGHr+0LZ4fsBwBfnSwTAAIMc6BgiDjVWaKxf1VvNhD+3PL9+FHRKfqhzY4PppEM3E/ecGH7mTDh6lw==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/types": "^3.973.6", @@ -603,6 +1340,71 @@ "node": ">=18.0.0" } }, + "node_modules/@aws-sdk/client-cloudfront": { + "version": "3.1136.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudfront/-/client-cloudfront-3.1136.0.tgz", + "integrity": "sha512-6HOtib5K0v85sVNcWEFma/uVUPkXi12pm2C2RenM+tt+noUVBF1CuySo+sDTPGuuHxB38GYnxAuKa7LfkgEV6Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/credential-provider-node": "^3.972.83", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-cloudfront/node_modules/@smithy/types": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.18.0.tgz", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/client-cloudwatch": { + "version": "3.1136.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudwatch/-/client-cloudwatch-3.1136.0.tgz", + "integrity": "sha512-DHFKsAYnnDxXhzPnQ7ef9LEdIdW89aCFmmTbHfpizg4jWI2wPvC2ZV2fv4IYdv5Ivk0XNp49kwQOeWbMGgnzpg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/credential-provider-node": "^3.972.83", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/middleware-compression": "^4.6.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-cloudwatch/node_modules/@smithy/types": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.18.0.tgz", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@aws-sdk/client-cognito-identity-provider": { "version": "3.1090.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity-provider/-/client-cognito-identity-provider-3.1090.0.tgz", @@ -1981,7 +2783,6 @@ "integrity": "sha512-FAORK6KoMQbd2VyLq/BMwcViy1txYd7XD9eYd5IGrXFpoOgWrSjp4zaSDlFPIEGgm68+n8fN0RelkbuMHCkSsg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-my-way-ts": "^0.1.5", "multipasta": "^0.2.5" @@ -2030,7 +2831,6 @@ "deprecated": "this package has been merged into the main effect package", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-check": "^3.21.0" }, @@ -3014,19 +3814,93 @@ "dev": true, "license": "BSD-3-Clause" }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "dev": true, - "license": "BSD-3-Clause" + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@puppeteer/browsers": { + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.2.tgz", + "integrity": "sha512-5EUZSUIc37H6aIXyWO0Z4y8NlF8NnjgmqeQgOGiswAU7pY0HOo16ho4+alIWmSfdZnjqBRawMsP3I5YqLSn6kw==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.4", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@puppeteer/browsers/node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@puppeteer/browsers/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@puppeteer/browsers/node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", - "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", - "dev": true, - "license": "BSD-3-Clause" + "node_modules/@puppeteer/browsers/node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.63.1", @@ -3468,6 +4342,33 @@ "node": ">=14.0.0" } }, + "node_modules/@smithy/middleware-compression": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-compression/-/middleware-compression-4.6.2.tgz", + "integrity": "sha512-Q9d+luiRjyHT6kCL/9NyGpdZJgodh4vtvfHC6H8SqoVKrV2k9RoyI9/IloVfdCYks3/2DIi7BsYYLcda5KZS0A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "fflate": "0.8.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/middleware-compression/node_modules/@smithy/types": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.18.0.tgz", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@smithy/node-http-handler": { "version": "4.12.1", "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", @@ -3573,6 +4474,19 @@ "node": ">=14.0.0" } }, + "node_modules/@sparticuz/chromium": { + "version": "143.0.4", + "resolved": "https://registry.npmjs.org/@sparticuz/chromium/-/chromium-143.0.4.tgz", + "integrity": "sha512-/6I7uQTRhRDD2/gGPQ1Gkf+Dqk0RYDACPJDZfSzz0OWk4JmUTonNHPXbrn6UIklOHlnDLf8xAAzkOZKB/cJpLA==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "tar-fs": "^3.1.1" + }, + "engines": { + "node": ">=20.11.0" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -3595,6 +4509,12 @@ "url": "https://github.com/sponsors/Fuzzyma" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@turf/bbox": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@turf/bbox/-/bbox-6.5.0.tgz", @@ -3869,7 +4789,7 @@ "version": "24.13.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" @@ -3903,6 +4823,16 @@ "@types/node": "*" } }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.59.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.1.tgz", @@ -3942,7 +4872,6 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-5.59.1.tgz", "integrity": "sha512-nzjFAN8WEu6yPRDizIFyzAfgK7nybPodMNFGNH0M9tei2gYnYszRDqVA0xlnRjkl7Hkx2vYrEdb6fP2a21cG1g==", "dev": true, - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.59.1", "@typescript-eslint/types": "5.59.1", @@ -4213,13 +5142,16 @@ "integrity": "sha512-UYvAq/XCA7xoh1juWDYsq3W0WywOB+pz8cgVnE1b45ZfdMhBvHDrgmSFG3jXeZSr2tMTYLGHFHON+ekG05Jebg==", "license": "MIT" }, + "node_modules/abstractplay-backend-crons": { + "resolved": "crons", + "link": true + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4332,7 +5264,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -4341,7 +5272,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -4508,6 +5438,18 @@ "node": ">=12" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4515,6 +5457,21 @@ "dev": true, "license": "MIT" }, + "node_modules/available-typed-arrays": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", + "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", + "license": "MIT", + "dependencies": { + "possible-typed-array-names": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/aws-amplify": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/aws-amplify/-/aws-amplify-6.20.0.tgz", @@ -4541,17 +5498,195 @@ "node": ">=18.0.0" } }, + "node_modules/aws-lambda": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/aws-lambda/-/aws-lambda-1.0.7.tgz", + "integrity": "sha512-9GNFMRrEMG5y3Jvv+V4azWvc+qNWdWLTjDdhf/zgMlz8haaaLWv0xeAIWxz9PuWUBawsVxy0zZotjCdR3Xq+2w==", + "license": "MIT", + "dependencies": { + "aws-sdk": "^2.814.0", + "commander": "^3.0.2", + "js-yaml": "^3.14.1", + "watchpack": "^2.0.0-beta.10" + }, + "bin": { + "lambda": "bin/lambda" + } + }, + "node_modules/aws-lambda/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/aws-lambda/node_modules/js-yaml": { + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/aws-sdk": { + "version": "2.1693.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1693.0.tgz", + "integrity": "sha512-cJmb8xEnVLT+R6fBS5sn/EFJiX7tUnDaPtOPZ1vFbOJtd0fnZn/Ky2XGgsvvoeliWeH7mL3TWSX5zXXGSQV6gQ==", + "deprecated": "The AWS SDK for JavaScript (v2) has reached end-of-support, and no longer receives updates. Please migrate your code to use AWS SDK for JavaScript (v3). More info https://a.co/cUPnyil", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "util": "^0.12.4", + "uuid": "8.0.0", + "xml2js": "0.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-sdk/node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", + "license": "MIT", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/aws-sdk/node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", + "license": "BSD-3-Clause" + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/b4a": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.9.0.tgz", + "integrity": "sha512-dpfcF9fDNR6++cthXR67iyhgqWy9CBouAvIWhIntzBG6cvK/cnIPiZQjBwi/ZqjjBEDGfoNDtmB0kTjroOJ3pQ==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/bare-events": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz", + "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.8.1.tgz", + "integrity": "sha512-N1nnXdHZAOSstz0XiHikGS4HGMH4CnSwhqWdGQQMqqdvp4Jybm9sE3R1WVnpWVd4SFkc8ryPDBLViNLwiEqECg==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.28.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-path": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.1.2.tgz", + "integrity": "sha512-ZyKbsuuqK6Ag0K8pX6V5Txq6XeJRvY+wXucnFGRjiyVYP9YWDpIQugk/b+enRYrEYBJaqLzghRQpXPMR7341Nw==", + "license": "Apache-2.0" + }, + "node_modules/bare-stream": { + "version": "2.13.4", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.4.tgz", + "integrity": "sha512-PcrQ8lVLbiJscNm1Kez+Yp4Gy4AHGcN1lzwjvf5NybWen7VvEgUfyfnXYJ2zNqWnzOfCb1Abq6lH8ti0syQszA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.5.4.tgz", + "integrity": "sha512-Gxa7UVWBr0/edU1b+TJhn/AZvMQUj9OGspvYsaTYQrAbZA4BOTZGL3LiZxvD+CeMlDH4juwD84+eTAp/bLYW5g==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -4567,6 +5702,15 @@ } ] }, + "node_modules/basic-ftp": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.3.1.tgz", + "integrity": "sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bestzip": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/bestzip/-/bestzip-2.2.1.tgz", @@ -4693,7 +5837,6 @@ "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dev": true, "license": "MIT", "dependencies": { "base64-js": "^1.0.2", @@ -4705,7 +5848,6 @@ "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "dev": true, "license": "MIT", "engines": { "node": "*" @@ -4727,6 +5869,53 @@ "node": ">=8" } }, + "node_modules/call-bind": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", + "integrity": "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "get-intrinsic": "^1.3.0", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -4827,6 +6016,19 @@ "node": ">=0.10.0" } }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -4865,6 +6067,12 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/commander": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-3.0.2.tgz", + "integrity": "sha512-Gar0ASD4BDyKC4hl4DwHqDrmvjoxWKZigVnAbn5H1owvm4CxCPdb0HQDehwNYMJpla5+M2tPmPARzhtYuwpHow==", + "license": "MIT" + }, "node_modules/complex.js": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.3.tgz", @@ -5036,6 +6244,37 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/detect-libc": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", @@ -5049,6 +6288,12 @@ "node": ">=0.10" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1608973", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz", + "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", + "license": "BSD-3-Clause" + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -5073,6 +6318,20 @@ "node": ">=6.0.0" } }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/earcut": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/earcut/-/earcut-2.2.4.tgz", @@ -5093,7 +6352,6 @@ "integrity": "sha512-fm1CQXCs7GTMwpadZxwU5rnAb2bN6mbQpknq/pXBGCzSKkosNYb9k0wsoL0TfB10rSa5xOEZJjo77u/KLXSANg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" @@ -5103,18 +6361,34 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, "license": "MIT" }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "dependencies": { "once": "^1.4.0" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -5122,6 +6396,18 @@ "dev": true, "license": "MIT" }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", @@ -5236,7 +6522,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -5260,12 +6545,41 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, "node_modules/eslint": { "version": "8.39.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.39.0.tgz", "integrity": "sha512-mwiok6cy7KTW7rBpo05k6+p4YVZByLNjAZ/ACB9DRCu4YDRwjXI01tWHp6KAUWelsBetTxKK/2sHB0vdS8Z2Og==", "dev": true, - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.4.0", @@ -5385,6 +6699,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", @@ -5450,7 +6777,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -5464,6 +6790,15 @@ "node": ">=0.8.x" } }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -5504,6 +6839,41 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "dev": true }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/fast-check": { "version": "3.23.2", "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.23.2.tgz", @@ -5532,6 +6902,12 @@ "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -5678,6 +7054,15 @@ "reusify": "^1.0.4" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -5782,6 +7167,41 @@ "integrity": "sha512-Gz1EvfOneuFfk4yG458dJ3TLJ7gV19q3OM/vVvvHf7eT02Hm1DleB4edsia6ahbKgAYxO9gvyQ1ioWZR+a00Yw==", "license": "MIT" }, + "node_modules/follow-redirects": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz", + "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-each": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", + "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", + "license": "MIT", + "dependencies": { + "is-callable": "^1.2.7" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -5851,6 +7271,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/gaxios": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.3.0.tgz", @@ -5881,6 +7310,15 @@ "node": ">=18" } }, + "node_modules/generator-function": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/generator-function/-/generator-function-2.0.1.tgz", + "integrity": "sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/geojson-rbush": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/geojson-rbush/-/geojson-rbush-3.2.0.tgz", @@ -5898,12 +7336,48 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "license": "ISC", "engines": { "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -5917,6 +7391,29 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/glicko2-lite": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/glicko2-lite/-/glicko2-lite-4.0.0.tgz", @@ -6020,11 +7517,22 @@ "node": ">=14" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "node_modules/grapheme-splitter": { "version": "1.0.4", @@ -6182,6 +7690,57 @@ "node": ">=8" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/honeycomb-grid": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/honeycomb-grid/-/honeycomb-grid-4.1.5.tgz", @@ -6202,6 +7761,19 @@ "node": ">=4" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/https-proxy-agent": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.1.tgz", @@ -6275,7 +7847,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -6367,6 +7938,31 @@ "resolved": "https://registry.npmjs.org/ion-js/-/ion-js-5.2.0.tgz", "integrity": "sha512-2ip7YvjTifaFRXrPgs4lTd+h3KcrWg2vrQ+vKEL443vvavbK8blB/C1FScm18cTaKkgfAr95pp3JLtndCtWaDQ==" }, + "node_modules/ip-address": { + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-arguments": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", + "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -6380,6 +7976,18 @@ "node": ">=8" } }, + "node_modules/is-callable": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", + "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -6393,12 +8001,30 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" } }, + "node_modules/is-generator-function": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz", + "integrity": "sha512-upqt1SkGkODW9tsGNG5mtXTXtECizwtS2kA161M+gJPc1xdb/Ax629af6YrTwcOeQHbewrPNlE5Dx7kzvXTizA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.4", + "generator-function": "^2.0.0", + "get-proto": "^1.0.1", + "has-tostringtag": "^1.0.2", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -6430,6 +8056,24 @@ "node": ">=8" } }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -6443,6 +8087,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-typed-array": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", + "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", + "license": "MIT", + "dependencies": { + "which-typed-array": "^1.1.16" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-unsafe": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", @@ -6458,8 +8117,7 @@ "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "dev": true + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, "node_modules/isexe": { "version": "2.0.0", @@ -6473,6 +8131,15 @@ "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==", "license": "MIT" }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "license": "Apache-2.0", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/js-combinatorics": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/js-combinatorics/-/js-combinatorics-2.1.4.tgz", @@ -6743,6 +8410,15 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mathjs": { "version": "13.2.3", "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-13.2.3.tgz", @@ -6845,6 +8521,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mnemonist": { "version": "0.38.3", "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", @@ -6909,6 +8601,15 @@ "integrity": "sha512-Tj+HTDSJJKaZnfiuw+iaF9skdPpTo2GtEly5JHnWV/hfv2Qj/9RKsGISQtLh2ox3l5EAGw487hnBee0sIJ6v2g==", "dev": true }, + "node_modules/netmask": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.1.1.tgz", + "integrity": "sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-addon-api": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", @@ -6979,7 +8680,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -7077,6 +8777,67 @@ "node": ">=8" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7131,6 +8892,33 @@ "node": ">=8" } }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -7157,6 +8945,12 @@ "node": ">= 14.16" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -7187,6 +8981,15 @@ "splaytree": "^3.1.0" } }, + "node_modules/possible-typed-array-names": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", + "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/postcss": { "version": "8.5.26", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", @@ -7250,6 +9053,15 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/protobufjs": { "version": "7.6.5", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", @@ -7274,6 +9086,72 @@ "node": ">=12.0.0" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -7283,6 +9161,24 @@ "node": ">=6" } }, + "node_modules/puppeteer-core": { + "version": "24.43.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.43.1.tgz", + "integrity": "sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.2", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1608973", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.20.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -7300,6 +9196,15 @@ ], "license": "MIT" }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -7410,7 +9315,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -7572,11 +9476,34 @@ } ] }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==", + "license": "ISC" + }, "node_modules/seedrandom": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", @@ -7670,6 +9597,23 @@ "bluebird": "3.7.2" } }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -7714,6 +9658,63 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.10", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.10.tgz", + "integrity": "sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.1.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -7730,6 +9731,12 @@ "integrity": "sha512-4OM2BJgC5UzrhVnnJA4BkHKGtjXNzzUfpQjCO8I05xYPsfS/VuQDwjCGGMi8rYQilHEV4j8NBqTFbls/PZEE7A==", "license": "MIT" }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -7744,6 +9751,32 @@ "dev": true, "license": "MIT" }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, + "node_modules/streamx": { + "version": "2.28.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.1.tgz", + "integrity": "sha512-zEzXb0s5Cds7tqMH6rhZ05lcJydCWiQPEwiNngVqzsxCc962vLY4Uw+mW7od8kDH258k2Uz/JrOkdIAAhSh9VA==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -7757,7 +9790,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -7772,7 +9804,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -7827,6 +9858,32 @@ "node": ">=8" } }, + "node_modules/tar-fs": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.3.tgz", + "integrity": "sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-fs/node_modules/tar-stream": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.1.tgz", + "integrity": "sha512-nqsEO8zLZJvrOMdEwkA0QdCLFbetHMn95Zqu4fKwX+hkaTWJPZZOrxx/PwtxoK0MMGQmBQNRW3CPs8IFYQz4cQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", @@ -7844,6 +9901,24 @@ "node": ">=6" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -7917,7 +9992,6 @@ "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -8497,12 +10571,17 @@ "node": ">= 18" } }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", "dev": true, - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -8535,7 +10614,7 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/universalify": { @@ -8556,11 +10635,40 @@ "punycode": "^2.1.0" } }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", + "license": "MIT", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", + "license": "MIT" + }, "node_modules/urlsafe-base64": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz", "integrity": "sha512-RtuPeMy7c1UrHwproMZN9gN6kiZ0SvJwRaEzwZY0j9MypEkFqyBaKv176jvlPtg58Zh36bOkS0NFABXMHvvGCA==" }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -8586,7 +10694,6 @@ "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", @@ -9119,7 +11226,6 @@ "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -9213,6 +11319,18 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/watchpack": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", + "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/web-push": { "version": "3.6.3", "resolved": "https://registry.npmjs.org/web-push/-/web-push-3.6.3.tgz", @@ -9241,6 +11359,12 @@ "node": ">= 8" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "license": "Apache-2.0" + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -9272,6 +11396,27 @@ "node": ">= 8" } }, + "node_modules/which-typed-array": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.24.tgz", + "integrity": "sha512-wk4Mf4pR5mRP7eYuuTBCIQ9d0ud2Fv2jRLQpfgnRjbOxAFHmjKFValgTpitVKzJJS8ajnYQV2Du1SZ8j6b/EUQ==", + "license": "MIT", + "dependencies": { + "available-typed-arrays": "^1.0.7", + "call-bind": "^1.0.9", + "call-bound": "^1.0.4", + "for-each": "^0.3.5", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", @@ -9303,7 +11448,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -9320,14 +11464,12 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/ws": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", - "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -9360,11 +11502,32 @@ "node": ">=16.0.0" } }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -9405,6 +11568,16 @@ "node": ">=10" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", @@ -9463,6 +11636,15 @@ "engines": { "node": ">=0.10.0" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 017b4cf3..5a1d285c 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "build:layers": "node scripts/build-layers.mjs", "test:layers": "npm run build:layers && node scripts/smoke-layer-modules.mjs", - "lint": "eslint && npm run typecheck", + "lint": "eslint && npm run typecheck && npm run lint:crons", "build": "npm run lint", "deploy-dev": "serverless deploy", "deploy-prod": "serverless --stage prod deploy", @@ -34,8 +34,11 @@ "backfill-feedback-user-engagement": "tsx bin/backfill-feedback-user-engagement-indexes.mjs", "backfill-wishlist-bgg-images": "tsx bin/backfill-wishlist-bgg-images.mjs", "backfill-blank-user-names": "tsx bin/backfill-blank-user-names.mjs", - "sync-deps": "ap-install-deps --stage dev", - "sync-deps:prod": "ap-install-deps --stage prod" + "sync-deps": "ap-install-deps --stage dev && node scripts/sync-crons-ap-deps.mjs --stage dev", + "sync-deps:prod": "ap-install-deps --stage prod && node scripts/sync-crons-ap-deps.mjs --stage prod", + "lint:crons": "npm run lint -w abstractplay-backend-crons", + "test:crons": "npm run test -w abstractplay-backend-crons", + "test:crons:layers": "npm run test:layers -w abstractplay-backend-crons" }, "repository": { "type": "git", @@ -52,9 +55,12 @@ "npm": "11.6.2" }, "packageManager": "npm@11.6.2", + "workspaces": [ + "crons" + ], "dependencies": { - "@abstractplay/gameslib": "1.0.0-ci-35279226489.0", - "@abstractplay/recranks": "latest", + "@abstractplay/gameslib": "1.0.0-ci-35453221145.0", + "@abstractplay/recranks": "1.0.0-ci-35280155165.0", "@abstractplay/renderer": "1.0.0-ci-35278756813.0", "@aws-sdk/client-apigatewaymanagementapi": "3.1090.0", "@aws-sdk/client-cognito-identity-provider": "3.1090.0", diff --git a/scripts/sync-crons-ap-deps.mjs b/scripts/sync-crons-ap-deps.mjs new file mode 100644 index 00000000..aed8826f --- /dev/null +++ b/scripts/sync-crons-ap-deps.mjs @@ -0,0 +1,68 @@ +/** + * Keep crons/package.json and crons/ci-deps.* aligned with the root lockfile. + * ap-install-deps must run from repo root (workspace hoists node_modules there). + */ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { getLockfileVersions } from "@abstractplay/ap-deps-tools/lockfile-versions"; + +const AP = { + gameslib: "@abstractplay/gameslib", + renderer: "@abstractplay/renderer", + recranks: "@abstractplay/recranks", +}; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, ".."); +const CRONS_ROOT = path.join(ROOT, "crons"); + +const stage = process.argv.includes("--stage") + ? process.argv[process.argv.indexOf("--stage") + 1] + : "dev"; + +if (stage !== "dev" && stage !== "prod") { + console.error("usage: node scripts/sync-crons-ap-deps.mjs [--stage dev|prod]"); + process.exit(1); +} + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function writeJson(filePath, data) { + fs.writeFileSync(filePath, `${JSON.stringify(data, null, 2)}\n`, "utf8"); +} + +const rootManifest = readJson(path.join(ROOT, `ci-deps.${stage}.json`)); +const lockVersions = getLockfileVersions(ROOT, Object.values(AP)); + +const cronsPkgPath = path.join(CRONS_ROOT, "package.json"); +const cronsPkg = readJson(cronsPkgPath); +cronsPkg.dependencies = cronsPkg.dependencies ?? {}; + +for (const pkg of Object.values(AP)) { + const key = pkg.split("/").pop(); + const version = lockVersions[pkg] ?? rootManifest[key]; + if (version && pkg in cronsPkg.dependencies) { + cronsPkg.dependencies[pkg] = version; + } +} +writeJson(cronsPkgPath, cronsPkg); + +const out = { + updatedAt: new Date().toISOString(), + source: rootManifest.source ?? `sync-crons-ap-deps from ${stage}`, +}; +if (lockVersions[AP.renderer] ?? rootManifest.renderer) { + out.renderer = lockVersions[AP.renderer] ?? rootManifest.renderer; +} +if (lockVersions[AP.gameslib] ?? rootManifest.gameslib) { + out.gameslib = lockVersions[AP.gameslib] ?? rootManifest.gameslib; +} +if (lockVersions[AP.recranks] ?? rootManifest.recranks) { + out.recranks = lockVersions[AP.recranks] ?? rootManifest.recranks; +} +writeJson(path.join(CRONS_ROOT, `ci-deps.${stage}.json`), out); + +console.log(`sync-crons-ap-deps: updated crons for stage=${stage}`, out);