diff --git a/.github/scripts/ci-changes.sh b/.github/scripts/ci-changes.sh index 75264dce1c5..0e25d496970 100755 --- a/.github/scripts/ci-changes.sh +++ b/.github/scripts/ci-changes.sh @@ -5,7 +5,7 @@ # skip list) to `true`; a change under .github/, an empty diff, or any failure to work out the # diff sets all of them. # -# full integration and e2e: any file not allowlisted below +# full integration, e2e, e2e-build, e2e-mobile: any file not allowlisted below # test the Vitest shards: content, docs app, and desktop changes, plus `full` # build the app's `next build`: content changes, plus `full` # desktop_live the live Electron suite: any file outside its own skip list diff --git a/.github/scripts/ci-gate.sh b/.github/scripts/ci-gate.sh index c6dea95a416..c7e4572e48b 100755 --- a/.github/scripts/ci-gate.sh +++ b/.github/scripts/ci-gate.sh @@ -5,10 +5,10 @@ # # A skip passes only for a gated job whose `changes` flag is exactly `false`: # -# integration, e2e full -# test test -# build build -# desktop-live desktop_live +# integration, e2e, e2e-build, e2e-mobile full +# test test +# build build +# desktop-live desktop_live # # Every other skip fails: an ungated job (lint, `changes` itself), a gated job whose flag is # `true`, missing, or anything else (a failed or skipped `changes` leaves no outputs). A failure @@ -24,7 +24,8 @@ if ! jq -e 'type == "object"' <<< "$needs" > /dev/null 2>&1; then fi failed="$(jq -r ' - {"integration": "full", "e2e": "full", "test": "test", "build": "build", "desktop-live": "desktop_live"} as $gate + {"integration": "full", "e2e": "full", "e2e-build": "full", "e2e-mobile": "full", "test": "test", + "build": "build", "desktop-live": "desktop_live"} as $gate | (.changes.outputs // {}) as $flags | to_entries[] | select(.value.result != "success") diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index feb8925d30d..bccb9491102 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash -# Runs one end-to-end suite group over real HTTP, each against its own `next dev` app. +# Runs one end-to-end suite group over real HTTP, each against its own app: `next dev`, except +# mobile, which serves the production build the `e2e-build` job uploads with `next start` (the +# job extracts it into apps/sim/.next first). # # Usage: http-e2e.sh (run from apps/sim) # # The job provides DATABASE_URL, BETTER_AUTH_SECRET and ENCRYPTION_KEY; each group sets the rest of # its app's environment here. Reports and server logs land in $RUNNER_TEMP/e2e. # -# The first request cold-compiles the app under Turbopack, which takes 42-150s on CI runners, so -# the readiness deadline only has to catch a hung boot: an exited server fails immediately, and -# either way the server log tail lands in the job log. +# Under `next dev` the first request cold-compiles the app under Turbopack, which takes 42-150s on +# CI runners, so the readiness deadline only has to catch a hung boot: an exited server fails +# immediately, and either way the server log tail lands in the job log. # # Each app starts from an empty Turbopack dev cache: a cache written under other NEXT_PUBLIC_* # values, by a server that `next dev` SIGKILLs 100ms after SIGTERM, can panic Turbopack or wedge a @@ -30,13 +32,22 @@ realtime_pid='' realtime_tag='' realtime_log='' browsers_pid='' +# `dev` or `start`; a group sets it before start_app. +app_server=dev finish() { local status=$? if [ -n "$server_pid" ]; then bash "$GITHUB_WORKSPACE/.github/scripts/stop-session.sh" "$server_pid" "$app_tag" || status=1 wait "$server_pid" 2>/dev/null || true - if [ -n "$status_log" ]; then + if [ -n "$status_log" ] && [ "$app_server" = start ]; then + # `next start` logs no requests; RouteHandler's closing line for each API request carries + # its status (an unhandled error's line is a 500 without one) and duration. + jq -Rr 'fromjson? | objects | select(.module == "RouteHandler" and .duration != null + and ((.path // "") | startswith("/api/"))) + | "\(.method) \(.path) \(.status // 500) in \(.duration)ms"' \ + "$server_log" > "$status_log" + elif [ -n "$status_log" ]; then awk '/^ (GET|POST|PUT|PATCH|DELETE|HEAD) \/api\// { print }' "$server_log" > "$status_log" fi if [ "$status" -ne 0 ]; then @@ -69,9 +80,20 @@ start_app() { export NEXT_PUBLIC_APP_URL="http://127.0.0.1:$port" export BETTER_AUTH_URL="$NEXT_PUBLIC_APP_URL" export DISABLE_TELEMETRY=true NEXT_TELEMETRY_DISABLED=1 - rm -rf .next/dev - E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \ - --port "$port" > "$server_log" 2>&1 & + if [ "$app_server" = start ]; then + if [ ! -f .next/BUILD_ID ]; then + echo "::error::No production build in apps/sim/.next for next start." + exit 1 + fi + # Production logs only errors by default; INFO writes the RouteHandler lines the status log + # reads. + E2E_APP="$app_tag" LOG_LEVEL=INFO setsid node ../../node_modules/next/dist/bin/next start \ + --hostname 127.0.0.1 --port "$port" > "$server_log" 2>&1 & + else + rm -rf .next/dev + E2E_APP="$app_tag" setsid node ../../node_modules/next/dist/bin/next dev --hostname 127.0.0.1 \ + --port "$port" > "$server_log" 2>&1 & + fi server_pid=$! wait_ready "$server_pid" "$NEXT_PUBLIC_APP_URL/api/health" "$label app" } @@ -189,14 +211,16 @@ case "$group" in bun run test:desktop-inbox:e2e ;; + # MOBILE_E2E_BROWSER (chromium or webkit) narrows the job to one browser; unset runs both. mobile) + app_server=start export NEXT_PUBLIC_FORCE_HOSTED=false export INTERNAL_API_SECRET=mobile-http-ci-local-secret-at-least-32-characters - export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=12288" export NEXT_PUBLIC_SOCKET_URL=http://127.0.0.1:3025 SOCKET_SERVER_URL=http://127.0.0.1:3025 + read -ra browsers <<< "${MOBILE_E2E_BROWSER:-chromium webkit}" # The browsers install while the app boots, in their own process group so finish() can stop them. browsers_log="$report_dir/mobile-playwright-install.log" - setsid --wait bunx --no-install playwright install --with-deps chromium webkit > "$browsers_log" 2>&1 & + setsid --wait bunx --no-install playwright install --with-deps "${browsers[@]}" > "$browsers_log" 2>&1 & browsers_pid=$! start_app mobile 3024 'mobile browser' record-http-status start_realtime mobile 3025 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 747808c42e9..66e78161a70 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -172,14 +172,14 @@ jobs: # The suites exercise HTTP behavior rather than a provisioning path, so they run once, against # the production (migrate) path. Each group's app environment lives in http-e2e.sh. # - # SCIM's hosted suites and the mobile browser matrix use 8 vCPU; the other - # self-hosted suites fit on 4. + # SCIM's hosted suites use 8 vCPU; the other self-hosted suites fit on 4. The mobile browser + # suite runs in e2e-mobile below. e2e: name: e2e (${{ matrix.group }}) needs: changes if: needs.changes.outputs.full == 'true' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && matrix.runner || 'ubuntu-latest' }} - timeout-minutes: ${{ matrix.group == 'mobile' && 40 || 20 }} + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -192,8 +192,6 @@ jobs: runner: blacksmith-4vcpu-ubuntu-2404 - group: desktop-inbox runner: blacksmith-4vcpu-ubuntu-2404 - - group: mobile - runner: blacksmith-8vcpu-ubuntu-2404 services: postgres: image: pgvector/pgvector:pg17 @@ -252,6 +250,86 @@ jobs: if-no-files-found: ignore retention-days: 7 + # The mobile browser suite against the production build e2e-build uploads, served by + # `next start` (http-e2e.sh), one browser per job: Chromium and WebKit no longer run back to + # back, and no route waits on a dev compile. Gated like `e2e`, on a build that already ran. + # + # 8 vCPU: `next start`, Postgres, the realtime server and one Playwright browser would fit a 4 + # vCPU runner's 16 GB, but the suite is timing-sensitive, and CPU contention between the browser + # and the server is what turns a slow tap into a flake. + e2e-mobile: + name: e2e (mobile, ${{ matrix.browser }}) + needs: [changes, e2e-build] + if: needs.changes.outputs.full == 'true' + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + browser: [chromium, webkit] + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: sim_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d sim_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/sim_test + BETTER_AUTH_SECRET: http-e2e-ci-secret-at-least-32-characters + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' + + steps: + - *checkout-mirror + - *checkout-plain + + - name: Setup workspace + uses: ./.github/actions/setup + with: + provider: ${{ vars.CI_PROVIDER }} + + - name: Provision the database through migrations + working-directory: packages/db + run: bun run db:migrate + + - name: Download the production build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: next-build + path: ${{ runner.temp }} + + # A hosted build (NEXT_PUBLIC_APP_URL on sim.ai) prerenders every post; the build production + # images use prerenders none. The URL itself is no marker: next.config's redirects name + # www.sim.ai in every build. + - name: Extract the production build + run: | + tar -I zstd -xf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim + jq -e '[.routes[].srcRoute] | any(. == "/blog/[slug]" or . == "/library/[slug]") | not' \ + apps/sim/.next/prerender-manifest.json > /dev/null || + { echo "::error::The build prerendered posts (a hosted build) or has no prerender manifest."; exit 1; } + + - name: Run end-to-end suites + working-directory: apps/sim + env: + MOBILE_E2E_BROWSER: ${{ matrix.browser }} + run: bash ../../.github/scripts/http-e2e.sh mobile + + - name: Upload end-to-end reports and server logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: http-e2e-mobile-${{ matrix.browser }} + path: ${{ runner.temp }}/e2e/ + if-no-files-found: ignore + retention-days: 7 + # Desktop tools in the real Electron app against a local app, on its own runner: the # Electron app, the dev app and its realtime server together outgrow the e2e runner. desktop-live: @@ -724,6 +802,71 @@ jobs: TURBO_CACHE_DIR: .turbo run: bunx turbo run build --filter=@sim/app + # The production build e2e-mobile serves with `next start`, built in the public-env mode + # production images use: NEXT_PUBLIC_APP_URL is docker/app.Dockerfile's localhost default, so + # every route is dynamic and NEXT_PUBLIC_* are read at start. (Images run the standalone + # DOCKER_BUILD server; this is the plain `next build` output.) The `build` job above keeps the + # hosted value, under which `next build` prerenders every post. Gated like `e2e`; its own Turbo + # cache key, so the two builds never evict each other's entries. 16 vCPU for the same reason as + # `build`: a cold build peaks at 51 GB. + e2e-build: + name: build (e2e) + needs: changes + if: needs.changes.outputs.full == 'true' + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-16vcpu-ubuntu-2404' || 'linux-x64-8-core' }} + timeout-minutes: 25 + + steps: + - *checkout-mirror + - *checkout-plain + + - name: Setup workspace + uses: ./.github/actions/setup + with: + provider: ${{ vars.CI_PROVIDER }} + turbo-cache-key: turbo-cache-build-e2e + + - name: Check runner memory headroom + run: | + TOTAL_GB=$(awk '/MemTotal/ {printf "%d", $2/1048576}' /proc/meminfo) + echo "Runner memory: ${TOTAL_GB} GB" + if [ "$TOTAL_GB" -lt 40 ]; then + echo "::warning::Runner has ${TOTAL_GB} GB. A cold-cache build peaks ~51 GB, so this run may be OOM-killed (reported only as 'the runner has received a shutdown signal'). Warm/partial builds should still fit." + fi + + - name: Build application + env: + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' + NEXT_PUBLIC_APP_URL: 'http://localhost:3000' + DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' + STRIPE_SECRET_KEY: 'dummy_key_for_ci_only' + STRIPE_WEBHOOK_SECRET: 'dummy_secret_for_ci_only' + RESEND_API_KEY: 'dummy_key_for_ci_only' + AWS_REGION: 'us-west-2' + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only + TURBO_CACHE_DIR: .turbo + run: bunx turbo run build --filter=@sim/app + + # Everything `next start` reads; node_modules and the source tree come from e2e-mobile's own + # checkout and setup, at the same workspace path and lockfile. + - name: Package the build for e2e-mobile + run: >- + tar -I 'zstd -T0' -cf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim + --exclude .next/cache --exclude .next/trace .next + + # Already zstd-compressed, so stored without zip compression. `overwrite` lets a re-run of + # this job replace the earlier attempt's build; three days covers re-running a failed shard + # after a weekend. + - name: Upload the build for e2e-mobile + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: next-build + path: ${{ runner.temp }}/next-build.tar.zst + if-no-files-found: error + compression-level: 0 + overwrite: true + retention-days: 3 + # One status for every check above: the single check to require on a branch ruleset, so adding, # sharding or renaming a job never means editing the ruleset. .github/scripts/ci-gate.sh decides: # a skip passes only for a gated job whose `changes` flag is exactly `false`; any other skip (a @@ -733,7 +876,7 @@ jobs: # on a cancelled run would report a cancelled head commit as passing. ci: name: ci - needs: [changes, integration, e2e, desktop-live, lint, test, build] + needs: [changes, integration, e2e, e2e-mobile, desktop-live, lint, test, build, e2e-build] if: ${{ always() }} runs-on: *runner-2vcpu timeout-minutes: 5 diff --git a/apps/sim/scripts/test-mobile-e2e.ts b/apps/sim/scripts/test-mobile-e2e.ts index bc596fcdd6d..6ad3b8f7b0f 100644 --- a/apps/sim/scripts/test-mobile-e2e.ts +++ b/apps/sim/scripts/test-mobile-e2e.ts @@ -64,6 +64,14 @@ const startedAt = new Date().toISOString() const checks: CheckResult[] = [] const screenshots: string[] = [] const httpErrors: { browser: string; status: number; path: string }[] = [] +/** + * Every document load mounts the workspace socket provider, which mints one realtime token from a + * route capped at 30 per minute per client IP. The suite loads well over 30 documents a minute, all + * from loopback, so that route's own rate-limit response is expected here; it is recorded apart + * from `httpErrors` and must still carry `Retry-After`, and each browser must mint at least once. + */ +const SOCKET_TOKEN_PATH = '/api/auth/socket-token' +const socketTokenMints = new Map() const requestFailures: { browser: string viewport: string @@ -507,8 +515,17 @@ async function exerciseViewport( } if (pageErrors.length < 20) pageErrors.push(error.message) }) + const mints = socketTokenMints.get(browserName) ?? { minted: 0, rateLimited: 0 } + socketTokenMints.set(browserName, mints) page.on('response', (response) => { const url = new URL(response.url()) + if (url.origin === baseUrl.origin && url.pathname === SOCKET_TOKEN_PATH) { + if (response.ok()) mints.minted += 1 + if (response.status() === 429 && response.headers()['retry-after']) { + mints.rateLimited += 1 + return + } + } if (url.origin === baseUrl.origin && response.status() >= 400 && httpErrors.length < 100) { httpErrors.push({ browser: browserName, status: response.status(), path: url.pathname }) } @@ -1194,6 +1211,11 @@ try { await check('same-origin HTTP responses succeed', undefined, async () => { assert.deepEqual(httpErrors, []) }) + await check('each browser mints a realtime token', undefined, async () => { + for (const [browserName, mints] of socketTokenMints) { + assert(mints.minted > 0, `${browserName} never minted a realtime token`) + } + }) await mkdir(dirname(reportPath), { recursive: true }) await writeFile( reportPath, @@ -1205,6 +1227,7 @@ try { checks, screenshots, httpErrors, + socketTokenMints: Object.fromEntries(socketTokenMints), requestFailures, navigationDiagnostics, viewportMeasurements, diff --git a/scripts/ci-gate.test.ts b/scripts/ci-gate.test.ts index 19ab78f6b8d..f51bd7160b6 100644 --- a/scripts/ci-gate.test.ts +++ b/scripts/ci-gate.test.ts @@ -5,10 +5,27 @@ import { describe, expect, it } from 'vitest' const script = fileURLToPath(new URL('../.github/scripts/ci-gate.sh', import.meta.url)) type Result = 'success' | 'failure' | 'cancelled' | 'skipped' -type Job = 'changes' | 'integration' | 'e2e' | 'desktop-live' | 'lint' | 'test' | 'build' +type Job = + | 'changes' + | 'integration' + | 'e2e' + | 'e2e-build' + | 'e2e-mobile' + | 'desktop-live' + | 'lint' + | 'test' + | 'build' type Results = Partial> -const GATED: Job[] = ['integration', 'e2e', 'desktop-live', 'test', 'build'] +const GATED: Job[] = [ + 'integration', + 'e2e', + 'e2e-build', + 'e2e-mobile', + 'desktop-live', + 'test', + 'build', +] const ALL_FALSE = { full: 'false', test: 'false', build: 'false', desktop_live: 'false' } /** @@ -80,7 +97,12 @@ describe('CI gate', () => { name: 'a skip whose flag is a misspelling', outputs: { ...ALL_FALSE, full: 'flase' }, results: skipGated, - reported: ['e2e: skipped', 'integration: skipped'], + reported: [ + 'e2e-build: skipped', + 'e2e-mobile: skipped', + 'e2e: skipped', + 'integration: skipped', + ], }, { name: 'a skip whose flag is differently cased', @@ -107,6 +129,8 @@ describe('CI gate', () => { reported: [ 'build: skipped', 'desktop-live: skipped', + 'e2e-build: skipped', + 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', 'test: skipped', @@ -119,6 +143,8 @@ describe('CI gate', () => { reported: [ 'build: skipped', 'desktop-live: skipped', + 'e2e-build: skipped', + 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', 'test: skipped', @@ -132,6 +158,8 @@ describe('CI gate', () => { 'build: skipped', 'changes: failure', 'desktop-live: skipped', + 'e2e-build: skipped', + 'e2e-mobile: skipped', 'e2e: skipped', 'integration: skipped', 'test: skipped', @@ -143,6 +171,12 @@ describe('CI gate', () => { results: { ...skipGated, changes: 'skipped' }, reported: ['changes: skipped'], }, + { + name: 'a mobile skip after a failed e2e build, though its flag is true', + outputs: { ...ALL_FALSE, full: 'true', build: 'true' }, + results: { 'e2e-build': 'failure', 'e2e-mobile': 'skipped' }, + reported: ['e2e-build: failure', 'e2e-mobile: skipped'], + }, { name: 'a failed job whose flag is false', outputs: ALL_FALSE,