Add the live Platform suite against the real stack

Author: waleedlatif1

.github/scripts/http-e2e.sh

@@ -3,7 +3,7 @@
 # CI groups, which reuse the production build from `e2e-build`.
 # The job extracts it into apps/sim/.next before running these suites.
 #
-# Usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|assistant-actions|organization-search|browser-extension|browser-extension-native|browser-extension-navigation|browser-extension-composer|browser-extension-access|project-files|mobile|seo>   (run from apps/sim)
+# Usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|assistant-actions|organization-search|browser-extension|browser-extension-native|browser-extension-navigation|browser-extension-composer|browser-extension-access|project-files|mobile|seo|platform-live-zone|platform-live-flag-off>   (run from apps/sim)
 #
 # The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of
 # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e.
@@ -20,7 +20,7 @@
 # telemetry flush runs detached and still writes .next/dev).
 set -euo pipefail
 
-group=${1:?usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|assistant-actions|organization-search|browser-extension|browser-extension-native|browser-extension-navigation|browser-extension-composer|browser-extension-access|project-files|mobile|seo>}
+group=${1:?usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|assistant-actions|organization-search|browser-extension|browser-extension-native|browser-extension-navigation|browser-extension-composer|browser-extension-access|project-files|mobile|seo|platform-live-zone|platform-live-flag-off>}
 report_dir="$RUNNER_TEMP/e2e"
 ready_timeout_seconds=600
 mkdir -p "$report_dir"
@@ -32,6 +32,10 @@ status_log=''
 realtime_pid=''
 realtime_tag=''
 realtime_log=''
+platform_pid=''
+platform_tag=''
+platform_log=''
+agent_pid=''
 browsers_pid=''
 local_app_url=''
 # `dev` or `start`; a group sets it before start_app.
@@ -63,6 +67,17 @@ finish() {
       tail -n 100 "$realtime_log"
     fi
   fi
+  if [ -n "$platform_pid" ]; then
+    bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$platform_pid" "$platform_tag" || status=1
+    wait "$platform_pid" 2>/dev/null || true
+    if [ "$status" -ne 0 ]; then
+      tail -n 100 "$platform_log"
+    fi
+  fi
+  if [ -n "$agent_pid" ]; then
+    kill "$agent_pid" 2>/dev/null || true
+    wait "$agent_pid" 2>/dev/null || true
+  fi
   if [ -n "$browsers_pid" ]; then
     kill -- "-$browsers_pid" 2>/dev/null || true
     wait "$browsers_pid" 2>/dev/null || true
@@ -133,6 +148,73 @@ start_realtime() {
   wait_ready "$realtime_pid" "http://127.0.0.1:$port/health" "$name realtime server"
 }
 
+# stop_app: stops the app start_app started, so a group can start it again under other flags.
+stop_app() {
+  bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$app_tag"
+  wait "$server_pid" 2>/dev/null || true
+  server_pid=''
+}
+
+# start_platform <port>: the production build of apps/sim/platform, behind the main app's rewrite.
+start_platform() {
+  local port=$1
+  platform_tag="platform-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$$"
+  platform_log="$report_dir/platform-live-platform.log"
+  E2E_APP="$platform_tag" SIM_PLATFORM_API_ORIGIN="$local_app_url" setsid node \
+    ../../node_modules/next/dist/bin/next start platform --hostname 127.0.0.1 --port "$port" \
+    > "$platform_log" 2>&1 &
+  platform_pid=$!
+  wait_ready "$platform_pid" "http://127.0.0.1:$port/platform" 'Platform zone'
+}
+
+# platform_live_environment: one hosted Enterprise organization, the deterministic seed its specs
+# read, and a scripted worker in place of the model. Hosted is decided by the app's hostname, which
+# the job maps to loopback. Google and Microsoft answer only to real credentials, so those clients
+# are fixtures, and what no fixture can reach is reported by the specs as a skip.
+platform_live_environment() {
+  app_server=start
+  export REDIS_URL=redis://127.0.0.1:6379
+  export E2E_APP_PUBLIC_URL=http://platform-live.sim.ai:3027
+  export INTERNAL_API_BASE_URL=http://127.0.0.1:3027 TRUSTED_ORIGINS=http://127.0.0.1:3027
+  export BILLING_ENABLED=true NEXT_PUBLIC_BILLING_ENABLED=true
+  export ENTERPRISE_ENABLED=true NEXT_PUBLIC_ENTERPRISE_ENABLED=true
+  export SSO_ENABLED=true NEXT_PUBLIC_SSO_ENABLED=true
+  export SCIM_ENABLED=true NEXT_PUBLIC_SCIM_ENABLED=true
+  export ORGANIZATIONS_ENABLED=true NEXT_PUBLIC_ORGANIZATIONS_ENABLED=true
+  export KNOWLEDGE_MEMBER_ACCESS=true CREDENTIAL_GROUPS=true ACCESS_CONTROL_ENABLED=true
+  export NEXT_PUBLIC_ACCESS_CONTROL_ENABLED=true
+  export SIM_LINKS=true PEOPLE=true
+  export INTERNAL_API_SECRET=platform-live-ci-local-secret-at-least-32-characters
+  export COPILOT_API_KEY=platform-live-ci-local-copilot-key MOTHERSHIP_SIM_TRANSPORT=direct
+  export SIM_AGENT_API_URL=http://127.0.0.1:3029
+  export GOOGLE_CLIENT_ID=platform-live-google-client GOOGLE_CLIENT_SECRET=platform-live-google-secret
+  export MICROSOFT_CLIENT_ID=platform-live-microsoft-client MICROSOFT_CLIENT_SECRET=platform-live-microsoft-secret
+  export STRIPE_SECRET_KEY=sk_test_platform_live STRIPE_WEBHOOK_SECRET=whsec_platform_live
+
+  export PLATFORM_LIVE_ORIGIN="$E2E_APP_PUBLIC_URL"
+  export PLATFORM_LIVE_PASSWORD=platform-live-password-1
+  export PLATFORM_LIVE_ORGANIZATION_ID=platform-live-organization
+  export PLATFORM_LIVE_ORGANIZATION_NAME='Sim Live Org'
+  export PLATFORM_LIVE_OWNER_EMAIL=olive@platform-live.test
+  export PLATFORM_LIVE_EMAIL=ada@platform-live.test
+  export PLATFORM_LIVE_MEMBER_EMAIL=mia@platform-live.test
+  export PLATFORM_LIVE_CONNECTED_EMAIL=cory@platform-live.test
+  export PLATFORM_LIVE_ROLE_TARGET_EMAIL=rae@platform-live.test
+  export PLATFORM_LIVE_REMOVABLE_EMAIL=remy@platform-live.test
+  export PLATFORM_LIVE_NO_DIRECTORY_EMAIL=noor@platform-live.test
+  export PLATFORM_LIVE_REFUSED_INVITE_EMAIL=otto@platform-live.test
+  export PLATFORM_LIVE_ORCHESTRATOR=1
+  bun --no-env-file scripts/platform-live-seed.ts
+}
+
+# run_platform_live <project> <report-name> [playwright args...]
+run_platform_live() {
+  local project=$1 name=$2
+  shift 2
+  PLATFORM_LIVE_REPORT_PATH="$report_dir/$name/report.json" \
+    bun run test:platform:live --project="$project" ${PLATFORM_LIVE_GREP:+--grep "$PLATFORM_LIVE_GREP"} "$@"
+}
+
 # wait_ready <pid> <health-url> <label>
 wait_ready() {
   local pid=$1 url=$2 label=$3
@@ -349,6 +431,38 @@ JS
       bun run test:landing-seo:e2e
     ;;
 
+  # The zone as deployed: flag on, /platform rewritten to the Platform build. Then the same build
+  # with the flag off, which must put everyone back on the legacy surface. Both report, whichever
+  # fails.
+  platform-live-zone)
+    platform_live_environment
+    export SIM_PLATFORM_ORIGIN=http://127.0.0.1:3028
+    export NEXT_PUBLIC_APP_URL="$E2E_APP_PUBLIC_URL"
+    start_browser_install platform-live --only-shell chromium
+    PLATFORM_ORG_UI=true start_app platform-live 3027 'Platform main' record-http-status
+    start_platform 3028
+    bun --no-env-file platform/checks/live/support/scripted-agent.ts 3029 \
+      > "$report_dir/platform-live-agent.log" 2>&1 &
+    agent_pid=$!
+    wait_for_browsers
+    live_status=0
+    run_platform_live zone platform-live || live_status=$?
+    stop_app
+    start_app platform-live-flag-off 3027 'Platform main, flag off' record-http-status
+    PLATFORM_LIVE_ZONE=1 run_platform_live flag-off platform-live-flag-off || live_status=$?
+    exit "$live_status"
+    ;;
+
+  # No zone at all: built without SIM_PLATFORM_ORIGIN, so even with the flag on nothing may change.
+  platform-live-flag-off)
+    platform_live_environment
+    export PLATFORM_ORG_UI=true
+    start_browser_install platform-live --only-shell chromium
+    start_app platform-live-flag-off 3027 'Main, no zone' record-http-status
+    wait_for_browsers
+    run_platform_live flag-off platform-live-flag-off
+    ;;
+
   *)
     echo "::error::Unknown end-to-end group: $group" >&2
     exit 2

.github/workflows/platform-design.yml

@@ -10,7 +10,17 @@ on:
       - 'scripts/sync-platform-*.ts'
       - '.github/workflows/platform-design.yml'
       - 'bun.lock'
+      - '.github/scripts/http-e2e.sh'
   workflow_dispatch:
+    inputs:
+      live_only:
+        description: Run only the live jobs, skipping the fixture suites.
+        type: boolean
+        default: false
+      grep:
+        description: Narrow the live suite to the specs matching this pattern.
+        type: string
+        default: ''
 
 permissions:
   contents: read
@@ -21,6 +31,7 @@ concurrency:
 
 jobs:
   browser:
+    if: ${{ !inputs.live_only }}
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     steps:
@@ -44,6 +55,7 @@ jobs:
           retention-days: 7
 
   desktop:
+    if: ${{ !inputs.live_only }}
     runs-on: macos-26
     timeout-minutes: 25
     steps:
@@ -69,3 +81,85 @@ jobs:
           name: platform-desktop
           path: ${{ runner.temp }}/platform-desktop
           retention-days: 7
+
+  # The Platform against the real stack: Postgres, Redis, migrations, the production main app and
+  # the production Platform. `zone` is the deployed topology, and then that same build with the flag
+  # off. `flag-off` builds with no zone at all, so it is the only proof /platform is truly absent.
+  # Group environment and seed: `.github/scripts/http-e2e.sh`.
+  live:
+    name: live (${{ matrix.topology }})
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }}
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        topology: [zone, flag-off]
+    services:
+      postgres:
+        image: pgvector/pgvector:pg17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: sim_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d sim_test"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:8.2-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test
+      BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters
+      ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000'
+      PLATFORM_LIVE_GREP: ${{ inputs.grep }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+          turbo-cache-key: turbo-cache-platform-live-${{ matrix.topology }}
+      - name: Provision the database through migrations
+        working-directory: packages/db
+        run: bun run db:migrate
+      - name: Build the main app
+        env:
+          NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
+          NEXT_PUBLIC_APP_URL: 'http://localhost:3000'
+          STRIPE_SECRET_KEY: 'dummy_key_for_ci_only'
+          STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only'
+          RESEND_API_KEY: 'dummy_key_for_ci_only'
+          AWS_REGION: 'us-west-2'
+          TURBO_CACHE_DIR: .turbo
+          NEXT_BUILD_SKIP_TYPE_CHECK: 'true'
+        run: |
+          if [ "${{ matrix.topology }}" = zone ]; then export SIM_PLATFORM_ORIGIN=http://127.0.0.1:3028; fi
+          bunx turbo run build --filter=@sim/app
+      - name: Build the Platform under its base path
+        if: matrix.topology == 'zone'
+        env:
+          SIM_PLATFORM_BASE_PATH: /platform
+        run: bun run --cwd apps/sim build:platform
+      - name: Map the hosted app to loopback
+        run: printf '127.0.0.1 platform-live.sim.ai\n' | sudo tee -a /etc/hosts
+      - name: Run the live suite
+        working-directory: apps/sim
+        run: bash ../../.github/scripts/http-e2e.sh platform-live-${{ matrix.topology }}
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        if: always()
+        with:
+          name: platform-live-${{ matrix.topology }}
+          path: ${{ runner.temp }}/e2e/
+          if-no-files-found: warn
+          retention-days: 7

apps/sim/platform/checks/live/chat.spec.ts

@@ -1,24 +1,11 @@
-import { expect, type Page, test } from '@playwright/test'
-
-function requireEnv(name: string): string {
-  const value = process.env[name]
-  if (!value) throw new Error(`${name} is required: the live checks use a real seeded member`)
-  return value
-}
-
-async function signIn(page: Page) {
-  await page.goto('/platform/home')
-  await page.locator('#email').fill(requireEnv('PLATFORM_LIVE_EMAIL'))
-  await page.locator('#password').fill(requireEnv('PLATFORM_LIVE_PASSWORD'))
-  await page.getByRole('button', { name: 'Sign in', exact: true }).click()
-  await expect(page).toHaveURL(/\/platform\//)
-}
+import { expect, test } from '@playwright/test'
+import { requireEnv, signInAsAdmin } from '@/platform/checks/live/session'
 
 /** A model turn needs a reachable orchestrator, which not every environment running these has. */
 const orchestrator = process.env.PLATFORM_LIVE_ORCHESTRATOR === '1'
 
 test('Home hands a draft to a new chat, which posts it for this organization', async ({ page }) => {
-  await signIn(page)
+  await signInAsAdmin(page, '/platform/home')
   const composer = page.getByRole('textbox', { name: /Ask about your business/ })
   await composer.fill('Say hello')
   const posted = page.waitForRequest(
@@ -36,22 +23,20 @@ test('Home hands a draft to a new chat, which posts it for this organization', a
 test('A chat outside the viewer’s access is not found rather than an empty chat', async ({
   page,
 }) => {
-  await signIn(page)
+  await signInAsAdmin(page, '/platform/home')
   const response = await page.goto('/platform/chat/00000000-0000-4000-8000-000000000000')
   expect(response?.status()).toBe(404)
 })
 
 test('A resource in the URL survives a reload of the chat route', async ({ page }) => {
-  await signIn(page)
-  await page.goto('/platform/chat?resource=stale-id')
+  await signInAsAdmin(page, '/platform/chat?resource=stale-id')
   await expect(page.getByRole('textbox')).toBeVisible()
   await expect(page).toHaveURL(/resource=stale-id/)
 })
 
 test('A running turn streams, stops, and keeps what arrived', async ({ page }) => {
   test.skip(!orchestrator, 'Needs PLATFORM_LIVE_ORCHESTRATOR=1')
-  await signIn(page)
-  await page.goto('/platform/chat')
+  await signInAsAdmin(page, '/platform/chat')
   await page.getByRole('textbox').fill('Count slowly from 1 to 50, one number per line.')
   await page.getByRole('button', { name: 'Send message' }).click()
   await expect(page).toHaveURL(/\/platform\/chat\/[^/?]+/, { timeout: 30_000 })
@@ -72,8 +57,7 @@ test('A message sent during a running turn waits in the queue and can be removed
   page,
 }) => {
   test.skip(!orchestrator, 'Needs PLATFORM_LIVE_ORCHESTRATOR=1')
-  await signIn(page)
-  await page.goto('/platform/chat')
+  await signInAsAdmin(page, '/platform/chat')
   const field = page.getByRole('textbox')
   await field.fill('Count slowly from 1 to 50, one number per line.')
   await page.getByRole('button', { name: 'Send message' }).click()
@@ -91,8 +75,7 @@ test('A gated tool holds the turn until the prompt is answered', async ({ page }
     !orchestrator || !process.env.PLATFORM_LIVE_APPROVAL_PROMPT,
     'Needs an orchestrator and a prompt that triggers a gated tool'
   )
-  await signIn(page)
-  await page.goto('/platform/chat')
+  await signInAsAdmin(page, '/platform/chat')
   await page.getByRole('textbox').fill(requireEnv('PLATFORM_LIVE_APPROVAL_PROMPT'))
   await page.getByRole('button', { name: 'Send message' }).click()
   const allow = page.getByRole('button', { name: 'Allow', exact: true })

apps/sim/platform/checks/live/flag-off/legacy-surface.spec.ts

@@ -0,0 +1,44 @@
+import { expect, type Page, test } from '@playwright/test'
+import { adminEmail, organizationId, signIn } from '@/platform/checks/live/session'
+
+/**
+ * `PLATFORM_LIVE_ZONE=1` means the app was built with `SIM_PLATFORM_ORIGIN` and is running with the
+ * flag off; unset, there is no zone at all, even with the flag on.
+ */
+const zoneBuilt = process.env.PLATFORM_LIVE_ZONE === '1'
+
+const LEGACY_ROUTES = ['home', 'settings/members', 'people'] as const
+
+async function expectLegacySurface(page: Page, route: string) {
+  const path = `/o/${organizationId()}/${route}`
+  const response = await page.goto(path)
+  expect(response?.status()).toBe(200)
+  expect(response?.request().redirectedFrom()).toBeNull()
+  await expect(page).toHaveURL(new RegExp(`${path}(?:\\?|$)`))
+  await expect(page.locator('.workspace-root')).toBeVisible()
+  await expect(page.locator('[data-platform-ui]')).toHaveCount(0)
+}
+
+for (const route of LEGACY_ROUTES) {
+  test(`/o/[id]/${route} renders the legacy surface with no redirect`, async ({ page }) => {
+    await signIn(page, adminEmail(), `/o/${organizationId()}/${route}`)
+    await expectLegacySurface(page, route)
+  })
+}
+
+test('a client navigation stays on the legacy surface too', async ({ page }) => {
+  await signIn(page, adminEmail(), `/o/${organizationId()}/home`)
+  await page.getByRole('link', { name: 'People', exact: true }).first().click()
+  await expect(page).toHaveURL(new RegExp(`/o/${organizationId()}/people`))
+  await expect(page.locator('[data-platform-ui]')).toHaveCount(0)
+})
+
+test('with no zone built, /platform is absent', async ({ page }) => {
+  test.skip(zoneBuilt, 'This build has a zone; its /platform is exercised by the zone project')
+  for (const path of ['/platform', '/platform/home', '/platform/settings/members']) {
+    await signIn(page, adminEmail(), `/o/${organizationId()}/home`)
+    const response = await page.goto(path)
+    expect(response?.status(), path).toBe(404)
+    await expect(page.locator('[data-platform-ui]')).toHaveCount(0)
+  }
+})