From bc27e4147cf1ff039dd90e0ca7228ab64c85925c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 15:48:42 -0700 Subject: [PATCH 1/4] ci: run the mobile e2e suite on the shared production build, one job per browser The build job now packages .next (zstd, cache excluded) and uploads it for a day. A new e2e-mobile job, one per browser (chromium, webkit), extracts it and serves it with `next start`, so the suite no longer waits on dev compiles or runs both browsers back to back. The mobile group leaves the e2e matrix. On runs that execute e2e, the build bakes NEXT_PUBLIC_APP_URL=http://localhost:3000, which is what production images bake, so the served build is the one that ships. Content-only pull requests keep the hosted value, under which next build prerenders every post. http-e2e.sh gains a `next start` mode used only by mobile: it installs only the selected browser, drops the dev heap override, and rebuilds the HTTP status log from RouteHandler's JSON lines at LOG_LEVEL=INFO. The ci gate treats e2e-mobile like e2e: a skip passes only when `full` is exactly false. --- .github/scripts/ci-changes.sh | 2 +- .github/scripts/ci-gate.sh | 11 ++-- .github/scripts/http-e2e.sh | 44 ++++++++++---- .github/workflows/checks.yml | 108 +++++++++++++++++++++++++++++++--- scripts/ci-gate.test.ts | 23 +++++++- 5 files changed, 162 insertions(+), 26 deletions(-) diff --git a/.github/scripts/ci-changes.sh b/.github/scripts/ci-changes.sh index 75264dce1c5..d07e463c884 100755 --- a/.github/scripts/ci-changes.sh +++ b/.github/scripts/ci-changes.sh @@ -5,7 +5,7 @@ # skip list) to `true`; a change under .github/, an empty diff, or any failure to work out the # diff sets all of them. # -# full integration and e2e: any file not allowlisted below +# full integration, e2e and e2e-mobile: any file not allowlisted below # test the Vitest shards: content, docs app, and desktop changes, plus `full` # build the app's `next build`: content changes, plus `full` # desktop_live the live Electron suite: any file outside its own skip list diff --git a/.github/scripts/ci-gate.sh b/.github/scripts/ci-gate.sh index c6dea95a416..ce64a84445d 100755 --- a/.github/scripts/ci-gate.sh +++ b/.github/scripts/ci-gate.sh @@ -5,10 +5,10 @@ # # A skip passes only for a gated job whose `changes` flag is exactly `false`: # -# integration, e2e full -# test test -# build build -# desktop-live desktop_live +# integration, e2e, e2e-mobile full +# test test +# build build +# desktop-live desktop_live # # Every other skip fails: an ungated job (lint, `changes` itself), a gated job whose flag is # `true`, missing, or anything else (a failed or skipped `changes` leaves no outputs). A failure @@ -24,7 +24,8 @@ if ! jq -e 'type == "object"' <<< "$needs" > /dev/null 2>&1; then fi failed="$(jq -r ' - {"integration": "full", "e2e": "full", "test": "test", "build": "build", "desktop-live": "desktop_live"} as $gate + {"integration": "full", "e2e": "full", "e2e-mobile": "full", "test": "test", "build": "build", + "desktop-live": "desktop_live"} as $gate | (.changes.outputs // {}) as $flags | to_entries[] | select(.value.result != "success") diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index feb8925d30d..805dff7d6ff 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash -# Runs one end-to-end suite group over real HTTP, each against its own `next dev` app. +# Runs one end-to-end suite group over real HTTP, each against its own app: `next dev`, except +# mobile, which serves the production build the `build` job uploads with `next start` (the job +# extracts it into apps/sim/.next first). # # Usage: http-e2e.sh (run from apps/sim) # # The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e. # -# The first request cold-compiles the app under Turbopack, which takes 42-150s on CI runners, so -# the readiness deadline only has to catch a hung boot: an exited server fails immediately, and -# either way the server log tail lands in the job log. +# Under `next dev` the first request cold-compiles the app under Turbopack, which takes 42-150s on +# CI runners, so the readiness deadline only has to catch a hung boot: an exited server fails +# immediately, and either way the server log tail lands in the job log. # # Each app starts from an empty Turbopack dev cache: a cache written under other NEXT_PUBLIC_* # values, by a server that `next dev` SIGKILLs 100ms after SIGTERM, can panic Turbopack or wedge a @@ -30,13 +32,22 @@ realtime_pid='' realtime_tag='' realtime_log='' browsers_pid='' +# `dev` or `start`; a group sets it before start_app. +app_server=dev finish() { local status=$? if [ -n "$server_pid" ]; then bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$app_tag" || status=1 wait "$server_pid" 2>/dev/null || true - if [ -n "$status_log" ]; then + if [ -n "$status_log" ] && [ "$app_server" = start ]; then + # `next start` logs no requests; RouteHandler's closing line for each API request carries + # its status (an unhandled error's line is a 500 without one) and duration. + jq -Rr 'fromjson? | objects | select(.module == "RouteHandler" and .duration != null + and ((.path // "") | startswith("/api/"))) + | "\(.method) \(.path) \(.status // 500) in \(.duration)ms"' \ + "$server_log" > "$status_log" + elif [ -n "$status_log" ]; then awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$status_log" fi if [ "$status" -ne 0 ]; then @@ -69,9 +80,20 @@ start_app() { export NEXT_PUBLIC_APP_URL="http://127.0.0.1:$port" export BETTER_AUTH_URL="$NEXT_PUBLIC_APP_URL" export DISABLE_TELEMETRY=true NEXT_TELEMETRY_DISABLED=1 - rm -rf .next/dev - E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \ - --port "$port" > "$server_log" 2>&1 & + if [ "$app_server" = start ]; then + if [ ! -f .next/BUILD_ID ]; then + echo "::error::No production build in apps/sim/.next for next start." + exit 1 + fi + # Production logs only errors by default; INFO writes the RouteHandler lines the status log + # reads. + E2E_APP="$app_tag" LOG_LEVEL=INFO setsid node ../../node_modules/next/dist/bin/next start \ + --hostname 127.0.0.1 --port "$port" > "$server_log" 2>&1 & + else + rm -rf .next/dev + E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \ + --port "$port" > "$server_log" 2>&1 & + fi server_pid=$! wait_ready "$server_pid" "$NEXT_PUBLIC_APP_URL/api/health" "$label app" } @@ -189,14 +211,16 @@ case "$group" in bun run test:desktop-inbox:e2e ;; + # MOBILE_E2E_BROWSER (chromium or webkit) narrows the job to one browser; unset runs both. mobile) + app_server=start export NEXT_PUBLIC_FORCE_HOSTED=false export INTERNAL_API_SECRET=mobile-http-ci-local-secret-at-least-32-characters - export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=12288" export NEXT_PUBLIC_SOCKET_URL=http://127.0.0.1:3025 SOCKET_SERVER_URL=http://127.0.0.1:3025 + read -ra browsers <<< "${MOBILE_E2E_BROWSER:-chromium webkit}" # The browsers install while the app boots, in their own process group so finish() can stop them. browsers_log="$report_dir/mobile-playwright-install.log" - setsid --wait bunx --no-install playwright install --with-deps chromium webkit > "$browsers_log" 2>&1 & + setsid --wait bunx --no-install playwright install --with-deps "${browsers[@]}" > "$browsers_log" 2>&1 & browsers_pid=$! start_app mobile 3024 'mobile browser' record-http-status start_realtime mobile 3025 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 747808c42e9..081e0b5bc9f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -172,14 +172,14 @@ jobs: # The suites exercise HTTP behavior rather than a provisioning path, so they run once, against # the production (migrate) path. Each group's app environment lives in http-e2e.sh. # - # SCIM's hosted suites and the mobile browser matrix use 8 vCPU; the other - # self-hosted suites fit on 4. + # SCIM's hosted suites use 8 vCPU; the other self-hosted suites fit on 4. The mobile browser + # suite runs in e2e-mobile below. e2e: name: e2e (${{ matrix.group }}) needs: changes if: needs.changes.outputs.full == 'true' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && matrix.runner || 'ubuntu-latest' }} - timeout-minutes: ${{ matrix.group == 'mobile' && 40 || 20 }} + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -192,8 +192,6 @@ jobs: runner: blacksmith-4vcpu-ubuntu-2404 - group: desktop-inbox runner: blacksmith-4vcpu-ubuntu-2404 - - group: mobile - runner: blacksmith-8vcpu-ubuntu-2404 services: postgres: image: pgvector/pgvector:pg17 @@ -252,6 +250,79 @@ jobs: if-no-files-found: ignore retention-days: 7 + # The mobile browser suite against the production build the `build` job uploads, served by + # `next start` (http-e2e.sh), one browser per job: Chromium and WebKit no longer run back to + # back, and no route waits on a dev compile. Gated like `e2e`, on a build that already ran. + # + # 8 vCPU: `next start`, Postgres, the realtime server and one Playwright browser would fit a 4 + # vCPU runner's 16 GB, but the suite is timing-sensitive, and CPU contention between the browser + # and the server is what turns a slow tap into a flake. + e2e-mobile: + name: e2e (mobile, ${{ matrix.browser }}) + needs: [changes, build] + if: needs.changes.outputs.full == 'true' + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + browser: [chromium, webkit] + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sim_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d sim_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test + BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' + + steps: + - *checkout-mirror + - *checkout-plain + + - name: Setup workspace + uses: ./.github/actions/setup + with: + provider: ${{ vars.CI_PROVIDER }} + + - name: Provision the database through migrations + working-directory: packages/db + run: bun run db:migrate + + - name: Download the production build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: next-build + path: ${{ runner.temp }} + + - name: Extract the production build + run: tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim + + - name: Run end-to-end suites + working-directory: apps/sim + env: + MOBILE_E2E_BROWSER: ${{ matrix.browser }} + run: bash ../../.github/scripts/http-e2e.sh mobile + + - name: Upload end-to-end reports and server logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: http-e2e-mobile-${{ matrix.browser }} + path: ${{ runner.temp }}/e2e/ + if-no-files-found: ignore + retention-days: 7 + # Desktop tools in the real Electron app against a local app, on its own runner: the # Electron app, the dev app and its realtime server together outgrow the e2e runner. desktop-live: @@ -714,7 +785,11 @@ jobs: - name: Build application env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' - NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' + # What production images bake (docker/app.Dockerfile), so the build e2e-mobile serves is + # the one that ships: every route dynamic, NEXT_PUBLIC_* read at start. A content-only + # pull request runs no e2e and keeps the hosted value, under which `next build` + # prerenders every post: the reason ci-changes.sh runs this job for content. + NEXT_PUBLIC_APP_URL: ${{ needs.changes.outputs.full == 'true' && 'http://localhost:3000' || 'https://www.sim.ai' }} DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' STRIPE_SECRET_KEY: 'dummy_key_for_ci_only' STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only' @@ -724,6 +799,25 @@ jobs: TURBO_CACHE_DIR: .turbo run: bunx turbo run build --filter=@sim/app + # Everything `next start` reads; node_modules and the source tree come from e2e-mobile's own + # checkout and setup, at the same workspace path and lockfile. + - name: Package the build for e2e-mobile + if: needs.changes.outputs.full == 'true' + run: tar -I 'zstd -T0' -cf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim --exclude .next/cache .next + + # Already zstd-compressed, so stored without zip compression. `overwrite` lets a re-run of + # this job replace the earlier attempt's build. + - name: Upload the build for e2e-mobile + if: needs.changes.outputs.full == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: next-build + path: ${{ runner.temp }}/next-build.tar.zst + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 1 + # One status for every check above: the single check to require on a branch ruleset, so adding, # sharding or renaming a job never means editing the ruleset. .github/scripts/ci-gate.sh decides: # a skip passes only for a gated job whose `changes` flag is exactly `false`; any other skip (a @@ -733,7 +827,7 @@ jobs: # on a cancelled run would report a cancelled head commit as passing. ci: name: ci - needs: [changes, integration, e2e, desktop-live, lint, test, build] + needs: [changes, integration, e2e, e2e-mobile, desktop-live, lint, test, build] if: ${{ always() }} runs-on: *runner-2vcpu timeout-minutes: 5 diff --git a/scripts/ci-gate.test.ts b/scripts/ci-gate.test.ts index 19ab78f6b8d..26a1d4fe8b1 100644 --- a/scripts/ci-gate.test.ts +++ b/scripts/ci-gate.test.ts @@ -5,10 +5,18 @@ import { describe, expect, it } from 'vitest' const script = fileURLToPath(new URL('../.github/scripts/ci-gate.sh', import.meta.url)) type Result = 'success' | 'failure' | 'cancelled' | 'skipped' -type Job = 'changes' | 'integration' | 'e2e' | 'desktop-live' | 'lint' | 'test' | 'build' +type Job = + | 'changes' + | 'integration' + | 'e2e' + | 'e2e-mobile' + | 'desktop-live' + | 'lint' + | 'test' + | 'build' type Results = Partial> -const GATED: Job[] = ['integration', 'e2e', 'desktop-live', 'test', 'build'] +const GATED: Job[] = ['integration', 'e2e', 'e2e-mobile', 'desktop-live', 'test', 'build'] const ALL_FALSE = { full: 'false', test: 'false', build: 'false', desktop_live: 'false' } /** @@ -80,7 +88,7 @@ describe('CI gate', () => { name: 'a skip whose flag is a misspelling', outputs: { ...ALL_FALSE, full: 'flase' }, results: skipGated, - reported: ['e2e: skipped', 'integration: skipped'], + reported: ['e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped'], }, { name: 'a skip whose flag is differently cased', @@ -107,6 +115,7 @@ describe('CI gate', () => { reported: [ 'build: skipped', 'desktop-live: skipped', + 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', 'test: skipped', @@ -119,6 +128,7 @@ describe('CI gate', () => { reported: [ 'build: skipped', 'desktop-live: skipped', + 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', 'test: skipped', @@ -132,6 +142,7 @@ describe('CI gate', () => { 'build: skipped', 'changes: failure', 'desktop-live: skipped', + 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', 'test: skipped', @@ -143,6 +154,12 @@ describe('CI gate', () => { results: { ...skipGated, changes: 'skipped' }, reported: ['changes: skipped'], }, + { + name: 'a mobile skip after a failed build, though its flag is true', + outputs: { ...ALL_FALSE, full: 'true', build: 'true' }, + results: { build: 'failure', 'e2e-mobile': 'skipped' }, + reported: ['build: failure', 'e2e-mobile: skipped'], + }, { name: 'a failed job whose flag is false', outputs: ALL_FALSE, From 95250161057bc27f3dd61d96fa7b3d4a74f2b369 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 15:56:06 -0700 Subject: [PATCH 2/4] ci: build mobile's artifact in its own job and keep the hosted build untouched The `build` job is back to staging byte for byte: its www.sim.ai build is CI's only render of every post, and it must stay on every run. A new `e2e-build` job, gated on `full` with its own Turbo cache key, builds with NEXT_PUBLIC_APP_URL=http://localhost:3000 (the Dockerfile default, so the same public-env mode production images use; images run the standalone server) and packages and uploads it for e2e-mobile, now kept 3 days and without .next/trace. e2e-mobile fails fast when the downloaded build prerendered posts, which only a hosted build does; the URL is no marker, since next.config's redirects name www.sim.ai in every build. The ci gate treats e2e-build like e2e. --- .github/scripts/ci-changes.sh | 2 +- .github/scripts/ci-gate.sh | 12 +++--- .github/scripts/http-e2e.sh | 4 +- .github/workflows/checks.yml | 77 ++++++++++++++++++++++++++++------- scripts/ci-gate.test.ts | 27 +++++++++--- 5 files changed, 94 insertions(+), 28 deletions(-) diff --git a/.github/scripts/ci-changes.sh b/.github/scripts/ci-changes.sh index d07e463c884..0e25d496970 100755 --- a/.github/scripts/ci-changes.sh +++ b/.github/scripts/ci-changes.sh @@ -5,7 +5,7 @@ # skip list) to `true`; a change under .github/, an empty diff, or any failure to work out the # diff sets all of them. # -# full integration, e2e and e2e-mobile: any file not allowlisted below +# full integration, e2e, e2e-build, e2e-mobile: any file not allowlisted below # test the Vitest shards: content, docs app, and desktop changes, plus `full` # build the app's `next build`: content changes, plus `full` # desktop_live the live Electron suite: any file outside its own skip list diff --git a/.github/scripts/ci-gate.sh b/.github/scripts/ci-gate.sh index ce64a84445d..c7e4572e48b 100755 --- a/.github/scripts/ci-gate.sh +++ b/.github/scripts/ci-gate.sh @@ -5,10 +5,10 @@ # # A skip passes only for a gated job whose `changes` flag is exactly `false`: # -# integration, e2e, e2e-mobile full -# test test -# build build -# desktop-live desktop_live +# integration, e2e, e2e-build, e2e-mobile full +# test test +# build build +# desktop-live desktop_live # # Every other skip fails: an ungated job (lint, `changes` itself), a gated job whose flag is # `true`, missing, or anything else (a failed or skipped `changes` leaves no outputs). A failure @@ -24,8 +24,8 @@ if ! jq -e 'type == "object"' <<< "$needs" > /dev/null 2>&1; then fi failed="$(jq -r ' - {"integration": "full", "e2e": "full", "e2e-mobile": "full", "test": "test", "build": "build", - "desktop-live": "desktop_live"} as $gate + {"integration": "full", "e2e": "full", "e2e-build": "full", "e2e-mobile": "full", "test": "test", + "build": "build", "desktop-live": "desktop_live"} as $gate | (.changes.outputs // {}) as $flags | to_entries[] | select(.value.result != "success") diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index 805dff7d6ff..bccb9491102 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Runs one end-to-end suite group over real HTTP, each against its own app: `next dev`, except -# mobile, which serves the production build the `build` job uploads with `next start` (the job -# extracts it into apps/sim/.next first). +# mobile, which serves the production build the `e2e-build` job uploads with `next start` (the +# job extracts it into apps/sim/.next first). # # Usage: http-e2e.sh (run from apps/sim) # diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 081e0b5bc9f..9a1deb1e383 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -250,7 +250,7 @@ jobs: if-no-files-found: ignore retention-days: 7 - # The mobile browser suite against the production build the `build` job uploads, served by + # The mobile browser suite against the production build e2e-build uploads, served by # `next start` (http-e2e.sh), one browser per job: Chromium and WebKit no longer run back to # back, and no route waits on a dev compile. Gated like `e2e`, on a build that already ran. # @@ -259,7 +259,7 @@ jobs: # and the server is what turns a slow tap into a flake. e2e-mobile: name: e2e (mobile, ${{ matrix.browser }}) - needs: [changes, build] + needs: [changes, e2e-build] if: needs.changes.outputs.full == 'true' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 25 @@ -305,8 +305,15 @@ jobs: name: next-build path: ${{ runner.temp }} + # A hosted build (NEXT_PUBLIC_APP_URL on sim.ai) prerenders every post; the build production + # images use prerenders none. The URL itself is no marker: next.config's redirects name + # www.sim.ai in every build. - name: Extract the production build - run: tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim + run: | + tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim + jq -e '.routes | keys | any(startswith("/blog/")) | not' \ + apps/sim/.next/prerender-manifest.json > /dev/null || + { echo "::error::The build prerendered posts (a hosted build) or has no prerender manifest."; exit 1; } - name: Run end-to-end suites working-directory: apps/sim @@ -785,11 +792,52 @@ jobs: - name: Build application env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' - # What production images bake (docker/app.Dockerfile), so the build e2e-mobile serves is - # the one that ships: every route dynamic, NEXT_PUBLIC_* read at start. A content-only - # pull request runs no e2e and keeps the hosted value, under which `next build` - # prerenders every post: the reason ci-changes.sh runs this job for content. - NEXT_PUBLIC_APP_URL: ${{ needs.changes.outputs.full == 'true' && 'http://localhost:3000' || 'https://www.sim.ai' }} + NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' + DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' + STRIPE_SECRET_KEY: 'dummy_key_for_ci_only' + STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only' + RESEND_API_KEY: 'dummy_key_for_ci_only' + AWS_REGION: 'us-west-2' + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only + TURBO_CACHE_DIR: .turbo + run: bunx turbo run build --filter=@sim/app + + # The production build e2e-mobile serves with `next start`, built in the public-env mode + # production images use: NEXT_PUBLIC_APP_URL is docker/app.Dockerfile's localhost default, so + # every route is dynamic and NEXT_PUBLIC_* are read at start. (Images run the standalone + # DOCKER_BUILD server; this is the plain `next build` output.) The `build` job above keeps the + # hosted value, under which `next build` prerenders every post. Gated like `e2e`; its own Turbo + # cache key, so the two builds never evict each other's entries. 16 vCPU for the same reason as + # `build`: a cold build peaks at 51 GB. + e2e-build: + name: build (e2e) + needs: changes + if: needs.changes.outputs.full == 'true' + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }} + timeout-minutes: 25 + + steps: + - *checkout-mirror + - *checkout-plain + + - name: Setup workspace + uses: ./.github/actions/setup + with: + provider: ${{ vars.CI_PROVIDER }} + turbo-cache-key: turbo-cache-build-e2e + + - name: Check runner memory headroom + run: | + TOTAL_GB=$(awk '/MemTotal/ {printf "%d", $2/1048576}' /proc/meminfo) + echo "Runner memory: ${TOTAL_GB} GB" + if [ "$TOTAL_GB" -lt 40 ]; then + echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit." + fi + + - name: Build application + env: + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' + NEXT_PUBLIC_APP_URL: 'http://localhost:3000' DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' STRIPE_SECRET_KEY: 'dummy_key_for_ci_only' STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only' @@ -802,13 +850,14 @@ jobs: # Everything `next start` reads; node_modules and the source tree come from e2e-mobile's own # checkout and setup, at the same workspace path and lockfile. - name: Package the build for e2e-mobile - if: needs.changes.outputs.full == 'true' - run: tar -I 'zstd -T0' -cf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim --exclude .next/cache .next + run: >- + tar -I 'zstd -T0' -cf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim + --exclude .next/cache --exclude .next/trace .next # Already zstd-compressed, so stored without zip compression. `overwrite` lets a re-run of - # this job replace the earlier attempt's build. + # this job replace the earlier attempt's build; three days covers re-running a failed shard + # after a weekend. - name: Upload the build for e2e-mobile - if: needs.changes.outputs.full == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: next-build @@ -816,7 +865,7 @@ jobs: if-no-files-found: error compression-level: 0 overwrite: true - retention-days: 1 + retention-days: 3 # One status for every check above: the single check to require on a branch ruleset, so adding, # sharding or renaming a job never means editing the ruleset. .github/scripts/ci-gate.sh decides: @@ -827,7 +876,7 @@ jobs: # on a cancelled run would report a cancelled head commit as passing. ci: name: ci - needs: [changes, integration, e2e, e2e-mobile, desktop-live, lint, test, build] + needs: [changes, integration, e2e, e2e-mobile, desktop-live, lint, test, build, e2e-build] if: ${{ always() }} runs-on: *runner-2vcpu timeout-minutes: 5 diff --git a/scripts/ci-gate.test.ts b/scripts/ci-gate.test.ts index 26a1d4fe8b1..f51bd7160b6 100644 --- a/scripts/ci-gate.test.ts +++ b/scripts/ci-gate.test.ts @@ -9,6 +9,7 @@ type Job = | 'changes' | 'integration' | 'e2e' + | 'e2e-build' | 'e2e-mobile' | 'desktop-live' | 'lint' @@ -16,7 +17,15 @@ type Job = | 'build' type Results = Partial> -const GATED: Job[] = ['integration', 'e2e', 'e2e-mobile', 'desktop-live', 'test', 'build'] +const GATED: Job[] = [ + 'integration', + 'e2e', + 'e2e-build', + 'e2e-mobile', + 'desktop-live', + 'test', + 'build', +] const ALL_FALSE = { full: 'false', test: 'false', build: 'false', desktop_live: 'false' } /** @@ -88,7 +97,12 @@ describe('CI gate', () => { name: 'a skip whose flag is a misspelling', outputs: { ...ALL_FALSE, full: 'flase' }, results: skipGated, - reported: ['e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped'], + reported: [ + 'e2e-build: skipped', + 'e2e-mobile: skipped', + 'e2e: skipped', + 'integration: skipped', + ], }, { name: 'a skip whose flag is differently cased', @@ -115,6 +129,7 @@ describe('CI gate', () => { reported: [ 'build: skipped', 'desktop-live: skipped', + 'e2e-build: skipped', 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', @@ -128,6 +143,7 @@ describe('CI gate', () => { reported: [ 'build: skipped', 'desktop-live: skipped', + 'e2e-build: skipped', 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', @@ -142,6 +158,7 @@ describe('CI gate', () => { 'build: skipped', 'changes: failure', 'desktop-live: skipped', + 'e2e-build: skipped', 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', @@ -155,10 +172,10 @@ describe('CI gate', () => { reported: ['changes: skipped'], }, { - name: 'a mobile skip after a failed build, though its flag is true', + name: 'a mobile skip after a failed e2e build, though its flag is true', outputs: { ...ALL_FALSE, full: 'true', build: 'true' }, - results: { build: 'failure', 'e2e-mobile': 'skipped' }, - reported: ['build: failure', 'e2e-mobile: skipped'], + results: { 'e2e-build': 'failure', 'e2e-mobile': 'skipped' }, + reported: ['e2e-build: failure', 'e2e-mobile: skipped'], }, { name: 'a failed job whose flag is false', From 795cf871d8284872997e48495d0e710fc1afa324 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 16:09:50 -0700 Subject: [PATCH 3/4] ci: key the non-hosted guard on prerendered post pages, not every /blog/ route --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9a1deb1e383..66e78161a70 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -311,7 +311,7 @@ jobs: - name: Extract the production build run: | tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim - jq -e '.routes | keys | any(startswith("/blog/")) | not' \ + jq -e '[.routes[].srcRoute] | any(. == "/blog/[slug]" or . == "/library/[slug]") | not' \ apps/sim/.next/prerender-manifest.json > /dev/null || { echo "::error::The build prerendered posts (a hosted build) or has no prerender manifest."; exit 1; } From 8173df90878ce2896164f189189387418b9ba6ff Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 12 Oct 2026 16:23:51 -0700 Subject: [PATCH 4/4] ci: expect the socket-token rate limit in the mobile e2e suite Each document load mints one realtime token from a route capped at 30/min per IP. On the production build the suite loads well over 30 documents a minute from loopback, so the route's designed 429 is recorded apart from httpErrors (only with Retry-After), and each browser must still mint at least once. --- apps/sim/scripts/test-mobile-e2e.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/apps/sim/scripts/test-mobile-e2e.ts b/apps/sim/scripts/test-mobile-e2e.ts index bc596fcdd6d..6ad3b8f7b0f 100644 --- a/apps/sim/scripts/test-mobile-e2e.ts +++ b/apps/sim/scripts/test-mobile-e2e.ts @@ -64,6 +64,14 @@ const startedAt = new Date().toISOString() const checks: CheckResult[] = [] const screenshots: string[] = [] const httpErrors: { browser: string; status: number; path: string }[] = [] +/** + * Every document load mounts the workspace socket provider, which mints one realtime token from a + * route capped at 30 per minute per client IP. The suite loads well over 30 documents a minute, all + * from loopback, so that route's own rate-limit response is expected here; it is recorded apart + * from `httpErrors` and must still carry `Retry-After`, and each browser must mint at least once. + */ +const SOCKET_TOKEN_PATH = '/api/auth/socket-token' +const socketTokenMints = new Map() const requestFailures: { browser: string viewport: string @@ -507,8 +515,17 @@ async function exerciseViewport( } if (pageErrors.length < 20) pageErrors.push(error.message) }) + const mints = socketTokenMints.get(browserName) ?? { minted: 0, rateLimited: 0 } + socketTokenMints.set(browserName, mints) page.on('response', (response) => { const url = new URL(response.url()) + if (url.origin === baseUrl.origin && url.pathname === SOCKET_TOKEN_PATH) { + if (response.ok()) mints.minted += 1 + if (response.status() === 429 && response.headers()['retry-after']) { + mints.rateLimited += 1 + return + } + } if (url.origin === baseUrl.origin && response.status() >= 400 && httpErrors.length < 100) { httpErrors.push({ browser: browserName, status: response.status(), path: url.pathname }) } @@ -1194,6 +1211,11 @@ try { await check('same-origin HTTP responses succeed', undefined, async () => { assert.deepEqual(httpErrors, []) }) + await check('each browser mints a realtime token', undefined, async () => { + for (const [browserName, mints] of socketTokenMints) { + assert(mints.minted > 0, `${browserName} never minted a realtime token`) + } + }) await mkdir(dirname(reportPath), { recursive: true }) await writeFile( reportPath, @@ -1205,6 +1227,7 @@ try { checks, screenshots, httpErrors, + socketTokenMints: Object.fromEntries(socketTokenMints), requestFailures, navigationDiagnostics, viewportMeasurements,