ci: run the mobile e2e suite on a production build, one job per browser

.github/scripts/ci-changes.sh

@@ -5,7 +5,7 @@
 # skip list) to `true`; a change under .github/, an empty diff, or any failure to work out the
 # diff sets all of them.
 #
-#   full          integration and e2e: any file not allowlisted below
+#   full          integration, e2e, e2e-build, e2e-mobile: any file not allowlisted below
 #   test          the Vitest shards: content, docs app, and desktop changes, plus `full`
 #   build         the app's `next build`: content changes, plus `full`
 #   desktop_live  the live Electron suite: any file outside its own skip list

.github/scripts/ci-gate.sh

@@ -5,10 +5,10 @@
 #
 # A skip passes only for a gated job whose `changes` flag is exactly `false`:
 #
-#   integration, e2e  full
-#   test              test
-#   build             build
-#   desktop-live      desktop_live
+#   integration, e2e, e2e-build, e2e-mobile  full
+#   test                                     test
+#   build                                    build
+#   desktop-live                             desktop_live
 #
 # Every other skip fails: an ungated job (lint, `changes` itself), a gated job whose flag is
 # `true`, missing, or anything else (a failed or skipped `changes` leaves no outputs). A failure
@@ -24,7 +24,8 @@ if ! jq -e 'type == "object"' <<< "$needs" > /dev/null 2>&1; then
 fi
 
 failed="$(jq -r '
-  {"integration": "full", "e2e": "full", "test": "test", "build": "build", "desktop-live": "desktop_live"} as $gate
+  {"integration": "full", "e2e": "full", "e2e-build": "full", "e2e-mobile": "full", "test": "test",
+   "build": "build", "desktop-live": "desktop_live"} as $gate
   | (.changes.outputs // {}) as $flags
   | to_entries[]
   | select(.value.result != "success")

.github/scripts/http-e2e.sh

@@ -1,14 +1,16 @@
 #!/usr/bin/env bash
-# Runs one end-to-end suite group over real HTTP, each against its own `next dev` app.
+# Runs one end-to-end suite group over real HTTP, each against its own app: `next dev`, except
+# mobile, which serves the production build the `e2e-build` job uploads with `next start` (the
+# job extracts it into apps/sim/.next first).
 #
 # Usage: http-e2e.sh <scim|cli|stop-after|desktop-inbox|mobile>   (run from apps/sim)
 #
 # The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of
 # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e.
 #
-# The first request cold-compiles the app under Turbopack, which takes 42-150s on CI runners, so
-# the readiness deadline only has to catch a hung boot: an exited server fails immediately, and
-# either way the server log tail lands in the job log.
+# Under `next dev` the first request cold-compiles the app under Turbopack, which takes 42-150s on
+# CI runners, so the readiness deadline only has to catch a hung boot: an exited server fails
+# immediately, and either way the server log tail lands in the job log.
 #
 # Each app starts from an empty Turbopack dev cache: a cache written under other NEXT_PUBLIC_*
 # values, by a server that `next dev` SIGKILLs 100ms after SIGTERM, can panic Turbopack or wedge a
@@ -30,13 +32,22 @@ realtime_pid=''
 realtime_tag=''
 realtime_log=''
 browsers_pid=''
+# `dev` or `start`; a group sets it before start_app.
+app_server=dev
 
 finish() {
   local status=$?
   if [ -n "$server_pid" ]; then
     bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$app_tag" || status=1
     wait "$server_pid" 2>/dev/null || true
-    if [ -n "$status_log" ]; then
+    if [ -n "$status_log" ] && [ "$app_server" = start ]; then
+      # `next start` logs no requests; RouteHandler's closing line for each API request carries
+      # its status (an unhandled error's line is a 500 without one) and duration.
+      jq -Rr 'fromjson? | objects | select(.module == "RouteHandler" and .duration != null
+        and ((.path // "") | startswith("/api/")))
+        | "\(.method) \(.path) \(.status // 500) in \(.duration)ms"' \
+        "$server_log" > "$status_log"
+    elif [ -n "$status_log" ]; then
       awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$status_log"
     fi
     if [ "$status" -ne 0 ]; then
@@ -69,9 +80,20 @@ start_app() {
   export NEXT_PUBLIC_APP_URL="http://127.0.0.1:$port"
   export BETTER_AUTH_URL="$NEXT_PUBLIC_APP_URL"
   export DISABLE_TELEMETRY=true NEXT_TELEMETRY_DISABLED=1
-  rm -rf .next/dev
-  E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \
-    --port "$port" > "$server_log" 2>&1 &
+  if [ "$app_server" = start ]; then
+    if [ ! -f .next/BUILD_ID ]; then
+      echo "::error::No production build in apps/sim/.next for next start."
+      exit 1
+    fi
+    # Production logs only errors by default; INFO writes the RouteHandler lines the status log
+    # reads.
+    E2E_APP="$app_tag" LOG_LEVEL=INFO setsid node ../../node_modules/next/dist/bin/next start \
+      --hostname 127.0.0.1 --port "$port" > "$server_log" 2>&1 &
+  else
+    rm -rf .next/dev
+    E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \
+      --port "$port" > "$server_log" 2>&1 &
+  fi
   server_pid=$!
   wait_ready "$server_pid" "$NEXT_PUBLIC_APP_URL/api/health" "$label app"
 }
@@ -189,14 +211,16 @@ case "$group" in
       bun run test:desktop-inbox:e2e
     ;;
 
+  # MOBILE_E2E_BROWSER (chromium or webkit) narrows the job to one browser; unset runs both.
   mobile)
+    app_server=start
     export NEXT_PUBLIC_FORCE_HOSTED=false
     export INTERNAL_API_SECRET=mobile-http-ci-local-secret-at-least-32-characters
-    export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=12288"
     export NEXT_PUBLIC_SOCKET_URL=http://127.0.0.1:3025 SOCKET_SERVER_URL=http://127.0.0.1:3025
+    read -ra browsers <<< "${MOBILE_E2E_BROWSER:-chromium webkit}"
     # The browsers install while the app boots, in their own process group so finish() can stop them.
     browsers_log="$report_dir/mobile-playwright-install.log"
-    setsid --wait bunx --no-install playwright install --with-deps chromium webkit > "$browsers_log" 2>&1 &
+    setsid --wait bunx --no-install playwright install --with-deps "${browsers[@]}" > "$browsers_log" 2>&1 &
     browsers_pid=$!
     start_app mobile 3024 'mobile browser' record-http-status
     start_realtime mobile 3025

.github/workflows/checks.yml

@@ -172,14 +172,14 @@ jobs:
   # The suites exercise HTTP behavior rather than a provisioning path, so they run once, against
   # the production (migrate) path. Each group's app environment lives in http-e2e.sh.
   #
-  # SCIM's hosted suites and the mobile browser matrix use 8 vCPU; the other
-  # self-hosted suites fit on 4.
+  # SCIM's hosted suites use 8 vCPU; the other self-hosted suites fit on 4. The mobile browser
+  # suite runs in e2e-mobile below.
   e2e:
     name: e2e (${{ matrix.group }})
     needs: changes
     if: needs.changes.outputs.full == 'true'
     runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && matrix.runner || 'ubuntu-latest' }}
-    timeout-minutes: ${{ matrix.group == 'mobile' && 40 || 20 }}
+    timeout-minutes: 20
     strategy:
       fail-fast: false
       matrix:
@@ -192,8 +192,6 @@ jobs:
             runner: blacksmith-4vcpu-ubuntu-2404
           - group: desktop-inbox
             runner: blacksmith-4vcpu-ubuntu-2404
-          - group: mobile
-            runner: blacksmith-8vcpu-ubuntu-2404
     services:
       postgres:
         image: pgvector/pgvector:pg17
@@ -252,6 +250,86 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
+  # The mobile browser suite against the production build e2e-build uploads, served by
+  # `next start` (http-e2e.sh), one browser per job: Chromium and WebKit no longer run back to
+  # back, and no route waits on a dev compile. Gated like `e2e`, on a build that already ran.
+  #
+  # 8 vCPU: `next start`, Postgres, the realtime server and one Playwright browser would fit a 4
+  # vCPU runner's 16 GB, but the suite is timing-sensitive, and CPU contention between the browser
+  # and the server is what turns a slow tap into a flake.
+  e2e-mobile:
+    name: e2e (mobile, ${{ matrix.browser }})
+    needs: [changes, e2e-build]
+    if: needs.changes.outputs.full == 'true'
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, webkit]
+    services:
+      postgres:
+        image: pgvector/pgvector:pg17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: sim_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d sim_test"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test
+      BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters
+      ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000'
+
+    steps:
+      - *checkout-mirror
+      - *checkout-plain
+
+      - name: Setup workspace
+        uses: ./.github/actions/setup
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+
+      - name: Provision the database through migrations
+        working-directory: packages/db
+        run: bun run db:migrate
+
+      - name: Download the production build
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: next-build
+          path: ${{ runner.temp }}
+
+      # A hosted build (NEXT_PUBLIC_APP_URL on sim.ai) prerenders every post; the build production
+      # images use prerenders none. The URL itself is no marker: next.config's redirects name
+      # www.sim.ai in every build.
+      - name: Extract the production build
+        run: |
+          tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim
+          jq -e '[.routes[].srcRoute] | any(. == "/blog/[slug]" or . == "/library/[slug]") | not' \
+            apps/sim/.next/prerender-manifest.json > /dev/null ||
+            { echo "::error::The build prerendered posts (a hosted build) or has no prerender manifest."; exit 1; }
+
+      - name: Run end-to-end suites
+        working-directory: apps/sim
+        env:
+          MOBILE_E2E_BROWSER: ${{ matrix.browser }}
+        run: bash ../../.github/scripts/http-e2e.sh mobile
+
+      - name: Upload end-to-end reports and server logs
+        if: failure()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: http-e2e-mobile-${{ matrix.browser }}
+          path: ${{ runner.temp }}/e2e/
+          if-no-files-found: ignore
+          retention-days: 7
+
   # Desktop tools in the real Electron app against a local app, on its own runner: the
   # Electron app, the dev app and its realtime server together outgrow the e2e runner.
   desktop-live:
@@ -724,6 +802,71 @@ jobs:
           TURBO_CACHE_DIR: .turbo
         run: bunx turbo run build --filter=@sim/app
 
+  # The production build e2e-mobile serves with `next start`, built in the public-env mode
+  # production images use: NEXT_PUBLIC_APP_URL is docker/app.Dockerfile's localhost default, so
+  # every route is dynamic and NEXT_PUBLIC_* are read at start. (Images run the standalone
+  # DOCKER_BUILD server; this is the plain `next build` output.) The `build` job above keeps the
+  # hosted value, under which `next build` prerenders every post. Gated like `e2e`; its own Turbo
+  # cache key, so the two builds never evict each other's entries. 16 vCPU for the same reason as
+  # `build`: a cold build peaks at 51 GB.
+  e2e-build:
+    name: build (e2e)
+    needs: changes
+    if: needs.changes.outputs.full == 'true'
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }}
+    timeout-minutes: 25
+
+    steps:
+      - *checkout-mirror
+      - *checkout-plain
+
+      - name: Setup workspace
+        uses: ./.github/actions/setup
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+          turbo-cache-key: turbo-cache-build-e2e
+
+      - name: Check runner memory headroom
+        run: |
+          TOTAL_GB=$(awk '/MemTotal/ {printf "%d", $2/1048576}' /proc/meminfo)
+          echo "Runner memory: ${TOTAL_GB} GB"
+          if [ "$TOTAL_GB" -lt 40 ]; then
+            echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit."
+          fi
+
+      - name: Build application
+        env:
+          NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
+          NEXT_PUBLIC_APP_URL: 'http://localhost:3000'
+          DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio'
+          STRIPE_SECRET_KEY: 'dummy_key_for_ci_only'
+          STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only'
+          RESEND_API_KEY: 'dummy_key_for_ci_only'
+          AWS_REGION: 'us-west-2'
+          ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only
+          TURBO_CACHE_DIR: .turbo
+        run: bunx turbo run build --filter=@sim/app
+
+      # Everything `next start` reads; node_modules and the source tree come from e2e-mobile's own
+      # checkout and setup, at the same workspace path and lockfile.
+      - name: Package the build for e2e-mobile
+        run: >-
+          tar -I 'zstd -T0' -cf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim
+          --exclude .next/cache --exclude .next/trace .next
+
+      # Already zstd-compressed, so stored without zip compression. `overwrite` lets a re-run of
+      # this job replace the earlier attempt's build; three days covers re-running a failed shard
+      # after a weekend.
+      - name: Upload the build for e2e-mobile
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: next-build
+          path: ${{ runner.temp }}/next-build.tar.zst
+          if-no-files-found: error
+          compression-level: 0
+          overwrite: true
+          retention-days: 3
+
   # One status for every check above: the single check to require on a branch ruleset, so adding,
   # sharding or renaming a job never means editing the ruleset. .github/scripts/ci-gate.sh decides:
   # a skip passes only for a gated job whose `changes` flag is exactly `false`; any other skip (a
@@ -733,7 +876,7 @@ jobs:
   # on a cancelled run would report a cancelled head commit as passing.
   ci:
     name: ci
-    needs: [changes, integration, e2e, desktop-live, lint, test, build]
+    needs: [changes, integration, e2e, e2e-mobile, desktop-live, lint, test, build, e2e-build]
     if: ${{ always() }}
     runs-on: *runner-2vcpu
     timeout-minutes: 5

apps/sim/scripts/test-mobile-e2e.ts

@@ -64,6 +64,14 @@ const startedAt = new Date().toISOString()
 const checks: CheckResult[] = []
 const screenshots: string[] = []
 const httpErrors: { browser: string; status: number; path: string }[] = []
+/**
+ * Every document load mounts the workspace socket provider, which mints one realtime token from a
+ * route capped at 30 per minute per client IP. The suite loads well over 30 documents a minute, all
+ * from loopback, so that route's own rate-limit response is expected here; it is recorded apart
+ * from `httpErrors` and must still carry `Retry-After`, and each browser must mint at least once.
+ */
+const SOCKET_TOKEN_PATH = '/api/auth/socket-token'
+const socketTokenMints = new Map<string, { minted: number; rateLimited: number }>()
 const requestFailures: {
   browser: string
   viewport: string
@@ -507,8 +515,17 @@ async function exerciseViewport(
     }
     if (pageErrors.length < 20) pageErrors.push(error.message)
   })
+  const mints = socketTokenMints.get(browserName) ?? { minted: 0, rateLimited: 0 }
+  socketTokenMints.set(browserName, mints)
   page.on('response', (response) => {
     const url = new URL(response.url())
+    if (url.origin === baseUrl.origin && url.pathname === SOCKET_TOKEN_PATH) {
+      if (response.ok()) mints.minted += 1
+      if (response.status() === 429 && response.headers()['retry-after']) {
+        mints.rateLimited += 1
+        return
+      }
+    }
     if (url.origin === baseUrl.origin && response.status() >= 400 && httpErrors.length < 100) {
       httpErrors.push({ browser: browserName, status: response.status(), path: url.pathname })
     }
@@ -1194,6 +1211,11 @@ try {
   await check('same-origin HTTP responses succeed', undefined, async () => {
     assert.deepEqual(httpErrors, [])
   })
+  await check('each browser mints a realtime token', undefined, async () => {
+    for (const [browserName, mints] of socketTokenMints) {
+      assert(mints.minted > 0, `${browserName} never minted a realtime token`)
+    }
+  })
   await mkdir(dirname(reportPath), { recursive: true })
   await writeFile(
     reportPath,
@@ -1205,6 +1227,7 @@ try {
         checks,
         screenshots,
         httpErrors,
+        socketTokenMints: Object.fromEntries(socketTokenMints),
         requestFailures,
         navigationDiagnostics,
         viewportMeasurements,