diff --git a/.github/scripts/check-project-rollout.py b/.github/scripts/check-project-rollout.py deleted file mode 100644 index a2f04734326..00000000000 --- a/.github/scripts/check-project-rollout.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -"""Read-only, fail-closed ECS retirement check for the Project column contract migration. - -The expected digest acknowledges fully deployed #8830 transaction-barrier and durable-authority support -and retirement of every incompatible server and relevant old worker job before switching. Every live and supported -rollback application must tolerate project_workspace being absent. -AWS checks below independently verify ECS retirement, not worker drainage. -""" -import argparse -import datetime -import json -import re -import subprocess -import sys - - -def aws(region, *args): - result = subprocess.run( - ['aws', '--region', region, '--no-cli-pager', '--cli-connect-timeout', '10', '--cli-read-timeout', '30', *args, '--output', 'json'], - capture_output=True, text=True, timeout=90, check=False, - ) - if result.returncode: - raise RuntimeError('AWS preflight read failed; check deployment-read permissions') - return json.loads(result.stdout) - - -def latest_execution(region, pipeline): - executions = aws(region, 'codepipeline', 'list-pipeline-executions', '--pipeline-name', pipeline).get('pipelineExecutionSummaries', []) - def epoch(execution): - value = execution['startTime'] - return value if isinstance(value, (int, float)) else datetime.datetime.fromisoformat(value.replace('Z', '+00:00')).timestamp() - if not executions: - raise RuntimeError('No application deployment execution was found') - return max(executions, key=epoch) - - -def matches_release(execution, digest): - return execution.get('status') == 'Succeeded' and any( - revision.get('actionName') == 'ECR_Source' and revision.get('revisionId') == digest - for revision in execution.get('sourceRevisions', []) - ) - - -def verify(environment, region, digest): - if not re.fullmatch(r'sha256:[0-9a-f]{64}', digest): - raise RuntimeError('Set the environment-specific PROJECT_COLUMN_ENFORCEMENT_READY_IMAGE_DIGEST after verifying #8830 authority-aware readers/writers, transaction barriers and all incompatible server/worker drainage') - pipeline = f'sim-{environment}-{region}-app-deployment' - execution = latest_execution(region, pipeline) - if not matches_release(execution, digest): - raise RuntimeError('The latest app pipeline has not completed for the acknowledged image; traffic cutover alone is insufficient') - execution_id = execution['pipelineExecutionId'] - group = aws(region, 'deploy', 'get-deployment-group', '--application-name', f'sim-{environment}-{region}-ecs-app', - '--deployment-group-name', f'sim-{environment}-{region}-app-dg')['deploymentGroupInfo'] - services = group.get('ecsServices', []) - if len(services) != 1: - raise RuntimeError('Expected exactly one application ECS service') - cluster, service = services[0]['clusterName'], services[0]['serviceName'] - description = aws(region, 'ecs', 'describe-services', '--cluster', cluster, '--services', service) - if description.get('failures') or len(description.get('services', [])) != 1: - raise RuntimeError('Cannot inspect the application ECS service') - record = description['services'][0] - if record.get('desiredCount', 0) < 1 or record.get('runningCount') != record['desiredCount'] or record.get('pendingCount') != 0: - raise RuntimeError('Application ECS service is not stable') - arns = set() - for status in ('RUNNING', 'STOPPED'): - arns.update(aws(region, 'ecs', 'list-tasks', '--cluster', cluster, '--service-name', service, - '--desired-status', status).get('taskArns', [])) - live = [] - ordered = sorted(arns) - for start in range(0, len(ordered), 100): - response = aws(region, 'ecs', 'describe-tasks', '--cluster', cluster, '--tasks', *ordered[start:start + 100]) - if response.get('failures'): - raise RuntimeError('Cannot account for every ECS task') - if len(response.get('tasks', [])) != len(ordered[start:start + 100]): - raise RuntimeError('Incomplete ECS task response') - live.extend(task for task in response['tasks'] if task.get('lastStatus') != 'STOPPED') - if len(live) != record['desiredCount']: - raise RuntimeError('Old, stopping, or pending ECS tasks remain') - for task in live: - app = [container for container in task.get('containers', []) if container.get('name') == 'app'] - if task.get('lastStatus') != 'RUNNING' or task.get('desiredStatus') != 'RUNNING' or len(app) != 1 or app[0].get('imageDigest') != digest: - raise RuntimeError('A live ECS task does not match the acknowledged compatible release') - latest = latest_execution(region, pipeline) - if latest.get('pipelineExecutionId') != execution_id or not matches_release(latest, digest): - raise RuntimeError('Application deployment changed during preflight') - print(json.dumps({'ecsRetired': True, 'expectedImageDigest': digest, 'pipelineExecutionId': execution_id, - 'operatorAcknowledgedColumnWritersAndWorkers': True})) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--environment', required=True, choices=['production', 'staging']) - parser.add_argument('--region', required=True) - parser.add_argument('--expected-image-digest', required=True) - args = parser.parse_args() - try: - verify(args.environment, args.region, args.expected_image_digest) - except (RuntimeError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired) as error: - print(f'Project column rollout preflight refused: {error}', file=sys.stderr) - sys.exit(1) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10f7de65385..c5de7794781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,7 +114,6 @@ jobs: migrate: permissions: contents: read - id-token: write name: migrate needs: [checks] # Explicit need results instead of the implicit success(): a skipped job @@ -134,10 +133,6 @@ jobs: STAGING_DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }} MIGRATION_DATABASE_URL: ${{ secrets.MIGRATION_DATABASE_URL }} STAGING_MIGRATION_DATABASE_URL: ${{ secrets.STAGING_MIGRATION_DATABASE_URL }} - AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} - STAGING_AWS_ROLE_TO_ASSUME: ${{ secrets.STAGING_AWS_ROLE_TO_ASSUME }} - AWS_REGION: ${{ secrets.AWS_REGION }} - STAGING_AWS_REGION: ${{ secrets.STAGING_AWS_REGION }} # Same ordering for dev (schema push before the dev image lands in ECR) migrate-dev: diff --git a/.github/workflows/migrate.yml b/.github/workflows/migrate.yml index 2a91393d41d..f0b04f4619d 100644 --- a/.github/workflows/migrate.yml +++ b/.github/workflows/migrate.yml @@ -18,14 +18,6 @@ on: required: false STAGING_MIGRATION_DATABASE_URL: required: false - AWS_ROLE_TO_ASSUME: - required: false - STAGING_AWS_ROLE_TO_ASSUME: - required: false - AWS_REGION: - required: false - STAGING_AWS_REGION: - required: false workflow_dispatch: inputs: environment: @@ -39,7 +31,6 @@ on: permissions: contents: read - id-token: write jobs: migrate: @@ -60,32 +51,13 @@ jobs: provider: ${{ vars.CI_PROVIDER }} node-version: '' - - name: Check Project column expansion and whether enforcement needs rollout evidence - id: project-contract + - name: Validate Project migration database prerequisites if: inputs.environment != 'dev' working-directory: ./packages/db env: DATABASE_URL: ${{ inputs.environment == 'production' && secrets.DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_DATABASE_URL || '' }} MIGRATION_DATABASE_URL: ${{ inputs.environment == 'production' && secrets.MIGRATION_DATABASE_URL || inputs.environment == 'staging' && secrets.STAGING_MIGRATION_DATABASE_URL || '' }} - run: bun --no-env-file scripts/project-contract-required.ts >> "$GITHUB_OUTPUT" - - - name: Configure AWS for Project connector retirement verification - if: steps.project-contract.outputs.required == 'true' - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b - with: - role-to-assume: ${{ inputs.environment == 'production' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} - aws-region: ${{ inputs.environment == 'production' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} - - # Acknowledge #8830 only after authority-aware readers/writers deploy and all incompatible - # servers and relevant old Trigger.dev runs drain. The runner commits the column-authority - # switch before backfill, validation, and connector retirement; retries never revert it. - # The preflight independently checks ECS retirement; it does not inspect worker runs. - - name: Require completed authority-aware rollout before Project cutover - if: steps.project-contract.outputs.required == 'true' - env: - ENVIRONMENT: ${{ inputs.environment }} - EXPECTED_IMAGE_DIGEST: ${{ inputs.environment == 'production' && vars.PROJECT_COLUMN_ENFORCEMENT_READY_IMAGE_DIGEST_PRODUCTION || inputs.environment == 'staging' && vars.PROJECT_COLUMN_ENFORCEMENT_READY_IMAGE_DIGEST_STAGING || '' }} - run: python3 .github/scripts/check-project-rollout.py --environment "$ENVIRONMENT" --region "$AWS_REGION" --expected-image-digest "$EXPECTED_IMAGE_DIGEST" + run: bun --no-env-file scripts/project-contract-required.ts # The expression maps the explicit environment input to exactly one repo # secret, so the job never holds another environment's database URL. An diff --git a/scripts/check-project-rollout.test.ts b/scripts/check-project-rollout.test.ts deleted file mode 100644 index c1052e7671b..00000000000 --- a/scripts/check-project-rollout.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { mkdtemp, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { delimiter, join, resolve } from 'node:path' -import { describe, expect, it } from 'vitest' - -const digest = `sha256:${'a'.repeat(64)}` -const otherDigest = `sha256:${'b'.repeat(64)}` -const execution = (id: string, time: number, revision: string, status = 'Succeeded') => ({ - pipelineExecutionId: id, - startTime: new Date(time * 1000).toISOString(), - status, - sourceRevisions: [{ actionName: 'ECR_Source', revisionId: revision }], -}) -const current = execution('current', 2, digest) -const older = execution('older', 1, otherDigest) -const pending = execution('pending', 3, otherDigest, 'InProgress') - -const cases = [ - { - name: 'accepts the latest completed acknowledged release in unordered history', - histories: [ - [older, current], - [older, current], - ], - allowed: true, - }, - { - name: 'rejects a stale acknowledgement after a different release succeeds', - histories: [[execution('different', 3, otherDigest), current]], - allowed: false, - }, - { - name: 'rejects a newer pending release even when an older success appears first', - histories: [[current, pending]], - allowed: false, - }, - { - name: 'rejects a newer deployment discovered in the final unordered history', - histories: [[current], [current, pending]], - allowed: false, - }, -] as const - -/** Exercises the preflight CLI against deployment histories without contacting an AWS account. */ -describe('Project column rollout release acknowledgement', () => { - it.each(cases)('$name', async ({ histories, allowed }) => { - const directory = await mkdtemp(join(tmpdir(), 'project-rollout-')) - try { - const fixture = join(directory, 'history.json') - await writeFile(fixture, JSON.stringify({ histories, digest })) - await writeFile( - join(directory, 'aws'), - `#!/usr/bin/env python3 -import json, os, pathlib, sys -fixture = pathlib.Path(os.environ['PROJECT_ROLLOUT_FIXTURE']) -data = json.loads(fixture.read_text()) -args = sys.argv -if 'list-pipeline-executions' in args: - counter = fixture.with_suffix('.count') - index = int(counter.read_text()) if counter.exists() else 0 - counter.write_text(str(index + 1)) - result = {'pipelineExecutionSummaries': data['histories'][min(index, len(data['histories']) - 1)]} -elif 'get-deployment-group' in args: - result = {'deploymentGroupInfo': {'ecsServices': [{'clusterName': 'fixture', 'serviceName': 'fixture'}]}} -elif 'describe-services' in args: - result = {'services': [{'desiredCount': 1, 'runningCount': 1, 'pendingCount': 0}]} -elif 'list-tasks' in args: - result = {'taskArns': ['fixture-task'] if args[args.index('--desired-status') + 1] == 'RUNNING' else []} -elif 'describe-tasks' in args: - result = {'tasks': [{'lastStatus': 'RUNNING', 'desiredStatus': 'RUNNING', 'containers': [{'name': 'app', 'imageDigest': data['digest']}]}]} -else: - raise SystemExit('Unexpected AWS command') -print(json.dumps(result)) -`, - { mode: 0o700 } - ) - const result = spawnSync( - 'python3', - [ - resolve('.github/scripts/check-project-rollout.py'), - '--environment', - 'staging', - '--region', - 'fixture-region', - '--expected-image-digest', - digest, - ], - { - encoding: 'utf8', - timeout: 10_000, - env: { - ...process.env, - PATH: `${directory}${delimiter}${process.env.PATH}`, - PROJECT_ROLLOUT_FIXTURE: fixture, - }, - } - ) - if (allowed) { - expect(result.status, result.stderr).toBe(0) - expect(JSON.parse(result.stdout)).toMatchObject({ - ecsRetired: true, - pipelineExecutionId: 'current', - operatorAcknowledgedColumnWritersAndWorkers: true, - }) - } else { - expect(result.status, result.stdout).toBe(1) - expect(result.stderr).toContain('Project column rollout preflight refused:') - } - } finally { - await rm(directory, { recursive: true, force: true }) - } - }) -})