From 5dd733528256d1c453946bb4cfc594ec85cdfcc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Mon, 19 Oct 2026 13:42:56 +0200 Subject: [PATCH 1/2] fix(auto-routing): address custom router review follow-ups Cap decision-token lifetime at verification, document fallback classification fields, check fallback entries against request capabilities, bound public request bodies, enforce the profile admission quota inside the admission batch, and resume unacknowledged grader usage on a retried client eval upload. --- .../src/custom-router.test.ts | 37 + .../src/custom-router.ts | 9 +- .../migrations/0025_complete_triton.sql | 1 + .../migrations/meta/0025_snapshot.json | 2157 +++++++++++++++++ .../migrations/meta/_journal.json | 7 + .../src/client-eval.test.ts | 19 + .../auto-routing-benchmark/src/client-eval.ts | 76 +- .../src/client-profiles.ts | 55 +- .../auto-routing-benchmark/src/db-schema.ts | 2 + .../src/profile-quota-sql.test.ts | 22 + .../src/profiles.test.ts | 6 + .../auto-routing-benchmark/src/profiles.ts | 107 +- .../src/custom-router-decision.ts | 17 +- .../auto-routing/src/custom-router.test.ts | 54 + services/auto-routing/src/decide.ts | 15 +- services/auto-routing/src/public-routes.ts | 22 +- 16 files changed, 2533 insertions(+), 73 deletions(-) create mode 100644 services/auto-routing-benchmark/migrations/0025_complete_triton.sql create mode 100644 services/auto-routing-benchmark/migrations/meta/0025_snapshot.json diff --git a/packages/auto-routing-contracts/src/custom-router.test.ts b/packages/auto-routing-contracts/src/custom-router.test.ts index 888cbd6e590..a60f929e15b 100644 --- a/packages/auto-routing-contracts/src/custom-router.test.ts +++ b/packages/auto-routing-contracts/src/custom-router.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { AUTO_DECISION_HEADER, + AUTO_DECISION_TTL_MS, CUSTOM_ROUTER_ERROR_HTTP_STATUS, ClientCustomRouterEntrySchema, ClientEvalPlanResponseSchema, @@ -279,6 +280,23 @@ describe('public decide API', () => { ).toBe(true); }); + it('keeps the classification on a fallback decision', () => { + const decision = { + entry: { kind: 'gateway', model: 'anthropic/claude-sonnet-4' }, + model: 'anthropic/claude-sonnet-4', + variant: null, + stale: false, + fallback: true, + sticky: false, + taskType: 'debugging', + subtaskType: 'bug_fixing', + tableVersion: null, + }; + expect( + CustomRouterDecideResponseSchema.parse({ decision, decisionId: 'v1.abc.def' }).decision + ).toMatchObject({ fallback: true, taskType: 'debugging', subtaskType: 'bug_fixing' }); + }); + it('maps every error code to a status', () => { for (const code of CustomRouterErrorCodeSchema.options) { expect(CUSTOM_ROUTER_ERROR_HTTP_STATUS[code]).toBeGreaterThanOrEqual(400); @@ -340,6 +358,25 @@ describe('auto decision tokens', () => { }); }); + it('rejects a token whose expiry exceeds the decision lifetime', async () => { + const nowMs = 1_000; + const longLived = await signAutoDecision( + { ...decisionPayload, expiresAt: nowMs + AUTO_DECISION_TTL_MS + 60_000 }, + DECISION_SECRET + ); + await expect(verifyAutoDecision(longLived, DECISION_SECRET, nowMs)).resolves.toEqual({ + ok: false, + reason: 'malformed', + }); + const fullLifetime = await signAutoDecision( + { ...decisionPayload, expiresAt: nowMs + AUTO_DECISION_TTL_MS }, + DECISION_SECRET + ); + await expect(verifyAutoDecision(fullLifetime, DECISION_SECRET, nowMs)).resolves.toMatchObject({ + ok: true, + }); + }); + it('rejects expired and malformed tokens', async () => { const expired = await signAutoDecision( { ...decisionPayload, expiresAt: 1_000 }, diff --git a/packages/auto-routing-contracts/src/custom-router.ts b/packages/auto-routing-contracts/src/custom-router.ts index d1c3b564c28..6e4a7263fdc 100644 --- a/packages/auto-routing-contracts/src/custom-router.ts +++ b/packages/auto-routing-contracts/src/custom-router.ts @@ -271,7 +271,8 @@ export const CustomRouterDecisionSchema = z model: z.string().min(1), variant: z.string().trim().min(1).nullable(), stale: z.boolean(), - // True when the classifier produced no decision and the default or first servable entry was used. + // True when no routing decision named a servable entry, so the default or first servable + // entry was used. taskType and subtaskType still report the classification, if any. fallback: z.boolean(), sticky: z.boolean(), taskType: ClassifierTaskTypeSchema.nullable(), @@ -296,6 +297,7 @@ export type CustomRouterDecideResponse = z.infer nowMs + AUTO_DECISION_TTL_MS + AUTO_DECISION_CLOCK_SKEW_MS) { + return { ok: false, reason: 'malformed' }; + } return { ok: true, payload: payload.data }; } diff --git a/services/auto-routing-benchmark/migrations/0025_complete_triton.sql b/services/auto-routing-benchmark/migrations/0025_complete_triton.sql new file mode 100644 index 00000000000..fec71921069 --- /dev/null +++ b/services/auto-routing-benchmark/migrations/0025_complete_triton.sql @@ -0,0 +1 @@ +ALTER TABLE `client_case_results` ADD `pending_grader_usage` text; \ No newline at end of file diff --git a/services/auto-routing-benchmark/migrations/meta/0025_snapshot.json b/services/auto-routing-benchmark/migrations/meta/0025_snapshot.json new file mode 100644 index 00000000000..e5a9b68f506 --- /dev/null +++ b/services/auto-routing-benchmark/migrations/meta/0025_snapshot.json @@ -0,0 +1,2157 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "f8ee87e8-b2e8-4aaa-87e6-fcc610bae23c", + "prevId": "09819ac0-dfe3-423d-91f4-ee97b6918685", + "tables": { + "benchmark_config": { + "name": "benchmark_config", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy_noise_band": { + "name": "accuracy_noise_band", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "frontier_min_accuracy_delta": { + "name": "frontier_min_accuracy_delta", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "production_token_profile": { + "name": "production_token_profile", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "switch_horizon_turns": { + "name": "switch_horizon_turns", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "low_confidence_threshold": { + "name": "low_confidence_threshold", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_concurrency": { + "name": "max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_max_concurrency": { + "name": "user_max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 100 + }, + "benchmark_user_id": { + "name": "benchmark_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "benchmark_org_id": { + "name": "benchmark_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "classifier_repetitions": { + "name": "classifier_repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "decider_repetitions": { + "name": "decider_repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "classifier_max_p95_latency_ms": { + "name": "classifier_max_p95_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "policy_overrides": { + "name": "policy_overrides", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "policy_computed": { + "name": "policy_computed", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "benchmark_config_router_unique": { + "name": "benchmark_config_router_unique", + "columns": [ + "router" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_profiles": { + "name": "benchmark_profiles", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_at": { + "name": "requested_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "platform_requested": { + "name": "platform_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "user_requested": { + "name": "user_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "benchmark_profiles_router_model_variant_engine_identity_repetitions_pk": { + "columns": [ + "router", + "model", + "variant", + "engine_identity", + "repetitions" + ], + "name": "benchmark_profiles_router_model_variant_engine_identity_repetitions_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_runs": { + "name": "benchmark_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy_noise_band": { + "name": "accuracy_noise_band", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "frontier_min_accuracy_delta": { + "name": "frontier_min_accuracy_delta", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "max_concurrency": { + "name": "max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "benchmark_user_id": { + "name": "benchmark_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "benchmark_org_id": { + "name": "benchmark_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "classifier_max_p95_latency_ms": { + "name": "classifier_max_p95_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'platform'" + } + }, + "indexes": { + "UQ_benchmark_runs_one_running_per_router_kind_purpose": { + "name": "UQ_benchmark_runs_one_running_per_router_kind_purpose", + "columns": [ + "router", + "kind", + "purpose" + ], + "isUnique": true, + "where": "\"benchmark_runs\".\"status\" = 'running'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "case_results": { + "name": "case_results", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fallback_reason": { + "name": "fallback_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "route_hit": { + "name": "route_hit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retried": { + "name": "retried", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_prefix": { + "name": "output_prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "event_count": { + "name": "event_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_event_types": { + "name": "last_event_types", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "timed_out": { + "name": "timed_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "prompt_revision": { + "name": "prompt_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "case_results_run_id_model_variant_case_id_rep_pk": { + "columns": [ + "run_id", + "model", + "variant", + "case_id", + "rep" + ], + "name": "case_results_run_id_model_variant_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "client_case_results": { + "name": "client_case_results", + "columns": { + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timed_out": { + "name": "timed_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "grader_cost_usd": { + "name": "grader_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "pending_grader_usage": { + "name": "pending_grader_usage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "client_case_results_profile_id_client_profiles_id_fk": { + "name": "client_case_results_profile_id_client_profiles_id_fk", + "tableFrom": "client_case_results", + "tableTo": "client_profiles", + "columnsFrom": [ + "profile_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "client_case_results_profile_id_case_id_rep_pk": { + "columns": [ + "profile_id", + "case_id", + "rep" + ], + "name": "client_case_results_profile_id_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "client_profiles": { + "name": "client_profiles", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entry_kind": { + "name": "entry_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "results": { + "name": "results", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errors": { + "name": "errors", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "timeouts": { + "name": "timeouts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "p50_latency_ms": { + "name": "p50_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "p95_latency_ms": { + "name": "p95_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "route_summaries": { + "name": "route_summaries", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "upload_seq": { + "name": "upload_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "UQ_client_profiles_identity": { + "name": "UQ_client_profiles_identity", + "columns": [ + "owner_type", + "owner_id", + "router", + "entry_kind", + "provider", + "model", + "variant", + "engine_identity", + "repetitions" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_classifier_models": { + "name": "config_classifier_models", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "config_classifier_models_router_model_pk": { + "columns": [ + "router", + "model" + ], + "name": "config_classifier_models_router_model_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_decider_models": { + "name": "config_decider_models", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "config_decider_models_router_model_pk": { + "columns": [ + "router", + "model" + ], + "name": "config_decider_models_router_model_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_summaries": { + "name": "model_summaries", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avg_latency_ms": { + "name": "avg_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "p50_latency_ms": { + "name": "p50_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cases": { + "name": "cases", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "errors": { + "name": "errors", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "p95_latency_ms": { + "name": "p95_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timeouts": { + "name": "timeouts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "route_accuracy": { + "name": "route_accuracy", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "carried": { + "name": "carried", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "model_summaries_run_id_model_variant_route_key_pk": { + "columns": [ + "run_id", + "model", + "variant", + "route_key" + ], + "name": "model_summaries_run_id_model_variant_route_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pending_decider_outputs": { + "name": "pending_decider_outputs", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt_revision": { + "name": "prompt_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": {}, + "foreignKeys": { + "pending_decider_outputs_run_id_benchmark_runs_id_fk": { + "name": "pending_decider_outputs_run_id_benchmark_runs_id_fk", + "tableFrom": "pending_decider_outputs", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "pending_decider_outputs_run_id_model_variant_case_id_rep_pk": { + "columns": [ + "run_id", + "model", + "variant", + "case_id", + "rep" + ], + "name": "pending_decider_outputs_run_id_model_variant_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "profile_remeasures": { + "name": "profile_remeasures", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_at": { + "name": "requested_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "IDX_profile_remeasures_run": { + "name": "IDX_profile_remeasures_run", + "columns": [ + "run_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "profile_remeasures_router_model_variant_engine_identity_repetitions_pk": { + "columns": [ + "router", + "model", + "variant", + "engine_identity", + "repetitions" + ], + "name": "profile_remeasures_router_model_variant_engine_identity_repetitions_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "profile_request_events": { + "name": "profile_request_events", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "IDX_profile_request_events_owner_router_admitted": { + "name": "IDX_profile_request_events_owner_router_admitted", + "columns": [ + "owner_type", + "owner_id", + "router", + "admitted_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "routing_table_candidates": { + "name": "routing_table_candidates", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rank": { + "name": "rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cases": { + "name": "cases", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meets_threshold": { + "name": "meets_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "pricing": { + "name": "pricing", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "estimated_request_cost_usd": { + "name": "estimated_request_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "routing_table_candidates_run_id_route_key_rank_pk": { + "columns": [ + "run_id", + "route_key", + "rank" + ], + "name": "routing_table_candidates_run_id_route_key_rank_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "routing_tables": { + "name": "routing_tables", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "generated_at": { + "name": "generated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "production_token_profile": { + "name": "production_token_profile", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "switch_horizon_turns": { + "name": "switch_horizon_turns", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "low_confidence_threshold": { + "name": "low_confidence_threshold", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "effort_by_complexity": { + "name": "effort_by_complexity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_lane_failures": { + "name": "run_lane_failures", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "chunk": { + "name": "chunk", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "shard": { + "name": "shard", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "failed_at": { + "name": "failed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "run_lane_failures_run_id_model_variant_rep_chunk_shard_pk": { + "columns": [ + "run_id", + "model", + "variant", + "rep", + "chunk", + "shard" + ], + "name": "run_lane_failures_run_id_model_variant_rep_chunk_shard_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_models": { + "name": "run_models", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "enqueued": { + "name": "enqueued", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "run_models_run_id_model_variant_pk": { + "columns": [ + "run_id", + "model", + "variant" + ], + "name": "run_models_run_id_model_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_chunks": { + "name": "scheduler_chunks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providers": { + "name": "providers", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk": { + "name": "chunk", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "case_ids": { + "name": "case_ids", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot": { + "name": "slot", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "available_at": { + "name": "available_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "IX_scheduler_chunks_pending": { + "name": "IX_scheduler_chunks_pending", + "columns": [ + "status", + "available_at" + ], + "isUnique": false + }, + "IX_scheduler_chunks_run": { + "name": "IX_scheduler_chunks_run", + "columns": [ + "run_id" + ], + "isUnique": false + }, + "IX_scheduler_chunks_lane": { + "name": "IX_scheduler_chunks_lane", + "columns": [ + "status", + "run_id", + "model", + "variant", + "rep", + "available_at", + "chunk" + ], + "isUnique": false + }, + "UQ_scheduler_chunks_live_slot": { + "name": "UQ_scheduler_chunks_live_slot", + "columns": [ + "slot" + ], + "isUnique": true, + "where": "\"scheduler_chunks\".\"status\" IN ('queued', 'active')" + } + }, + "foreignKeys": { + "scheduler_chunks_run_id_benchmark_runs_id_fk": { + "name": "scheduler_chunks_run_id_benchmark_runs_id_fk", + "tableFrom": "scheduler_chunks", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_cooldowns": { + "name": "scheduler_cooldowns", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "until": { + "name": "until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "scheduler_cooldowns_scope_key_pk": { + "columns": [ + "scope", + "key" + ], + "name": "scheduler_cooldowns_scope_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_dispatch_lock": { + "name": "scheduler_dispatch_lock", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_executions": { + "name": "scheduler_executions", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "prompt_revision": { + "name": "prompt_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": {}, + "foreignKeys": { + "scheduler_executions_run_id_benchmark_runs_id_fk": { + "name": "scheduler_executions_run_id_benchmark_runs_id_fk", + "tableFrom": "scheduler_executions", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "scheduler_executions_run_id_model_variant_rep_case_id_pk": { + "columns": [ + "run_id", + "model", + "variant", + "rep", + "case_id" + ], + "name": "scheduler_executions_run_id_model_variant_rep_case_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_runs": { + "name": "scheduler_runs", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "scheduler_runs_run_id_benchmark_runs_id_fk": { + "name": "scheduler_runs_run_id_benchmark_runs_id_fk", + "tableFrom": "scheduler_runs", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_settings": { + "name": "scheduler_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slots": { + "name": "slots", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_concurrency": { + "name": "model_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_concurrency": { + "name": "provider_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "unknown_provider_concurrency": { + "name": "unknown_provider_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/services/auto-routing-benchmark/migrations/meta/_journal.json b/services/auto-routing-benchmark/migrations/meta/_journal.json index 65029f40a20..69a96feda65 100644 --- a/services/auto-routing-benchmark/migrations/meta/_journal.json +++ b/services/auto-routing-benchmark/migrations/meta/_journal.json @@ -176,6 +176,13 @@ "when": 1792216595824, "tag": "0024_married_vanisher", "breakpoints": true + }, + { + "idx": 25, + "version": "6", + "when": 1792409925260, + "tag": "0025_complete_triton", + "breakpoints": true } ] } \ No newline at end of file diff --git a/services/auto-routing-benchmark/src/client-eval.test.ts b/services/auto-routing-benchmark/src/client-eval.test.ts index 337c1eb3509..53cc1663d69 100644 --- a/services/auto-routing-benchmark/src/client-eval.test.ts +++ b/services/auto-routing-benchmark/src/client-eval.test.ts @@ -444,6 +444,25 @@ describe('client eval (real SQLite)', () => { expect(second?.id).not.toBe(first?.id); expect(sqlite.prepare('SELECT COUNT(*) AS n FROM client_case_results').get()).toEqual({ n: 1 }); }); + + it('resends the stored grader usage on retry when removing an unbilled upload fails', async () => { + const judged = { caseId: judgedCase.id }; + sqlite.exec( + "CREATE TRIGGER keep_case_results BEFORE DELETE ON client_case_results BEGIN SELECT RAISE(ABORT, 'delete failed'); END" + ); + writeUsage.mockResolvedValueOnce({ kind: 'unavailable', reason: 'http_error' }); + await expect(gradeClientEvalResult(db, CONFIG, upload(judged), deps)).rejects.toThrow(); + + expect(await gradeClientEvalResult(db, CONFIG, upload(judged), deps)).toEqual({ score: 0.75 }); + expect(await gradeClientEvalResult(db, CONFIG, upload(judged), deps)).toEqual({ score: 0.75 }); + expect(writeUsage).toHaveBeenCalledTimes(2); + const [first, resent] = writeUsage.mock.calls.map(([usage]) => usage); + expect(resent).toEqual(first); + expect(judge).toHaveBeenCalledOnce(); + expect( + sqlite.prepare('SELECT pending_grader_usage AS pending FROM client_case_results').get() + ).toEqual({ pending: null }); + }); }); describe('clientEvalCase', () => { diff --git a/services/auto-routing-benchmark/src/client-eval.ts b/services/auto-routing-benchmark/src/client-eval.ts index 9b5a5886a8c..1ecc60d6357 100644 --- a/services/auto-routing-benchmark/src/client-eval.ts +++ b/services/auto-routing-benchmark/src/client-eval.ts @@ -22,13 +22,14 @@ import { type InternalClientEvalPlanRequest, type InternalClientEvalResultRequest, } from '@kilocode/auto-routing-contracts'; -import type { UsageRecordRequest } from '@kilocode/usage-contracts'; +import { UsageRecordRequestSchema, type UsageRecordRequest } from '@kilocode/usage-contracts'; import { formatError } from '@kilocode/worker-utils'; import type { UsageWriteOutcome } from '@kilocode/worker-utils/usage-writer-client'; import type { Hono } from 'hono'; import type * as z from 'zod'; import { DECIDER_CLI_TIMEOUT_MS, deciderCliPrompt } from './cli-runner'; import { + clearPendingGraderUsage, clientCaseKey, clientPairKey, deleteClientCaseUpload, @@ -211,6 +212,11 @@ export async function gradeClientEvalResult( const plannedResults = deciderCasesForRouter(request.baseRouter).length * current.repetitions; const stored = await getClientCase(db, caseKey); if (stored?.ran) { + if (stored.pendingGraderUsage !== null) { + const usage = UsageRecordRequestSchema.parse(JSON.parse(stored.pendingGraderUsage)); + if (!(await recordGraderUsage(deps, usage))) throw graderUsageNotRecorded(); + await clearPendingGraderUsage(db, { ...caseKey, revision: stored.revision }); + } await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: stored.score }; } @@ -227,6 +233,29 @@ export async function gradeClientEvalResult( throw new ClientEvalError('internal_error', 'The grader is unavailable; retry the upload.'); } + const usage = + grade.costUsd > 0 + ? await buildGraderUsage( + request.billing, + [ + request.billing.userId, + request.billing.organizationId, + request.ownerType, + request.ownerId, + request.baseRouter, + entryKey, + request.variant, + current.engineIdentity, + current.repetitions, + request.caseId, + request.rep, + ], + grade.costUsd, + nowIso + ).catch(() => { + throw graderUsageNotRecorded(); + }) + : null; const revision = await saveClientCaseResult(db, { profile_id: profile.id, case_id: request.caseId, @@ -240,50 +269,37 @@ export async function gradeClientEvalResult( score: grade.score, grader_cost_usd: grade.costUsd, uploaded_at: nowIso, + pending_grader_usage: usage === null ? null : JSON.stringify(usage), }); if (revision === null) { await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: (await getClientCase(db, caseKey))?.score ?? grade.score }; } - if (grade.costUsd > 0) { - const billed = await buildGraderUsage( - request.billing, - [ - request.billing.userId, - request.billing.organizationId, - request.ownerType, - request.ownerId, - request.baseRouter, - entryKey, - request.variant, - current.engineIdentity, - current.repetitions, - request.caseId, - request.rep, - ], - grade.costUsd, - nowIso - ) - .then(usage => deps.writeUsage(usage)) - .then( - outcome => outcome.kind !== 'unavailable', - () => false - ); - if (!billed) { + if (usage !== null) { + if (!(await recordGraderUsage(deps, usage))) { await deleteClientCaseUpload(db, { ...caseKey, revision }); await refreshClientProfileRollup(db, profile, plannedResults, nowIso); - throw new ClientEvalError( - 'internal_error', - 'Grader usage was not recorded; retry the upload.' - ); + throw graderUsageNotRecorded(); } + await clearPendingGraderUsage(db, { ...caseKey, revision }); } await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: grade.score }; } +function recordGraderUsage(deps: ClientEvalDeps, usage: UsageRecordRequest): Promise { + return deps.writeUsage(usage).then( + outcome => outcome.kind !== 'unavailable', + () => false + ); +} + +function graderUsageNotRecorded(): ClientEvalError { + return new ClientEvalError('internal_error', 'Grader usage was not recorded; retry the upload.'); +} + async function requireConfig(db: D1Database, router: AutoRouter): Promise { const config = await getBenchmarkConfig(db, router); if (!config) { diff --git a/services/auto-routing-benchmark/src/client-profiles.ts b/services/auto-routing-benchmark/src/client-profiles.ts index 7b8d0b96a4f..79a6bbfe3b6 100644 --- a/services/auto-routing-benchmark/src/client-profiles.ts +++ b/services/auto-routing-benchmark/src/client-profiles.ts @@ -144,12 +144,19 @@ const ranSuccessfully = sql`(${clientCaseResults.exit_code} IS NOT NULL AND ${cl export async function getClientCase( db: D1Database, key: { profileId: string; caseId: string; rep: number } -): Promise<{ score: number; ran: boolean } | null> { +): Promise<{ + score: number; + ran: boolean; + revision: number; + pendingGraderUsage: string | null; +} | null> { const [row] = await drizzle(db) .select({ score: clientCaseResults.score, exitCode: clientCaseResults.exit_code, timedOut: clientCaseResults.timed_out, + revision: clientCaseResults.revision, + pendingGraderUsage: clientCaseResults.pending_grader_usage, }) .from(clientCaseResults) .where( @@ -159,7 +166,14 @@ export async function getClientCase( eq(clientCaseResults.rep, key.rep) ) ); - return row ? { score: row.score, ran: row.exitCode === 0 && !row.timedOut } : null; + return row + ? { + score: row.score, + ran: row.exitCode === 0 && !row.timedOut, + revision: row.revision, + pendingGraderUsage: row.pendingGraderUsage, + } + : null; } /** @@ -194,6 +208,7 @@ export async function saveClientCaseResult( timed_out: row.timed_out, score: row.score, grader_cost_usd: row.grader_cost_usd, + pending_grader_usage: row.pending_grader_usage, uploaded_at: row.uploaded_at, revision: sql`excluded.revision`, }, @@ -208,17 +223,33 @@ export async function saveClientCaseResult( export async function deleteClientCaseUpload( db: D1Database, key: { profileId: string; caseId: string; rep: number; revision: number } +): Promise { + await drizzle(db).delete(clientCaseResults).where(clientCaseUploadWhere(key)); +} + +/** Mark one upload's grader usage as acknowledged by the usage writer. */ +export async function clearPendingGraderUsage( + db: D1Database, + key: { profileId: string; caseId: string; rep: number; revision: number } ): Promise { await drizzle(db) - .delete(clientCaseResults) - .where( - and( - eq(clientCaseResults.profile_id, key.profileId), - eq(clientCaseResults.case_id, key.caseId), - eq(clientCaseResults.rep, key.rep), - eq(clientCaseResults.revision, key.revision) - ) - ); + .update(clientCaseResults) + .set({ pending_grader_usage: null }) + .where(clientCaseUploadWhere(key)); +} + +function clientCaseUploadWhere(key: { + profileId: string; + caseId: string; + rep: number; + revision: number; +}) { + return and( + eq(clientCaseResults.profile_id, key.profileId), + eq(clientCaseResults.case_id, key.caseId), + eq(clientCaseResults.rep, key.rep), + eq(clientCaseResults.revision, key.revision) + ); } /** `${caseId}\0${rep}` keys of cases that ran, per profile id. */ @@ -255,7 +286,7 @@ export function clientCaseKey(caseId: string, rep: number): string { return `${caseId}\0${rep}`; } -type GradedClientCase = Omit; +type GradedClientCase = Omit; async function listGradedClientCases( db: D1Database, diff --git a/services/auto-routing-benchmark/src/db-schema.ts b/services/auto-routing-benchmark/src/db-schema.ts index 469d8846b02..1e39a7c56e1 100644 --- a/services/auto-routing-benchmark/src/db-schema.ts +++ b/services/auto-routing-benchmark/src/db-schema.ts @@ -540,6 +540,8 @@ export const clientCaseResults = sqliteTable( // This upload's revision from the profile's `upload_seq`, unique within the profile even // across same-millisecond uploads and delete-and-reinsert; the rollup guard compares it. revision: integer('revision').notNull().default(0), + // Grader usage the usage writer has not acknowledged yet; a retried upload resends it. + pending_grader_usage: text('pending_grader_usage'), }, table => [primaryKey({ columns: [table.profile_id, table.case_id, table.rep] })] ); diff --git a/services/auto-routing-benchmark/src/profile-quota-sql.test.ts b/services/auto-routing-benchmark/src/profile-quota-sql.test.ts index 52dd99723ec..6b6fcbddb75 100644 --- a/services/auto-routing-benchmark/src/profile-quota-sql.test.ts +++ b/services/auto-routing-benchmark/src/profile-quota-sql.test.ts @@ -71,4 +71,26 @@ describe('platform-paid admission quota (real SQLite)', () => { n: 0, }); }); + + it('admits only the remaining capacity when concurrent requests race at the limit', async () => { + const filled = models('filled', PROFILE_ADMISSION_LIMIT - 1); + for (let offset = 0; offset < filled.length; offset += MAX_POOL_ENTRIES) { + await admit('efficient', filled.slice(offset, offset + MAX_POOL_ENTRIES)); + } + + const results = await Promise.allSettled([ + admit('efficient', ['race/a']), + admit('efficient', ['race/b']), + ]); + + expect(results.map(result => result.status).toSorted()).toEqual(['fulfilled', 'rejected']); + const rejected = results.find(result => result.status === 'rejected'); + expect(rejected?.reason).toBeInstanceOf(ProfileQuotaExceededError); + expect(sqlite.prepare('SELECT COUNT(*) AS n FROM profile_request_events').get()).toEqual({ + n: PROFILE_ADMISSION_LIMIT, + }); + expect( + sqlite.prepare("SELECT COUNT(*) AS n FROM benchmark_profiles WHERE model LIKE 'race/%'").get() + ).toEqual({ n: 1 }); + }); }); diff --git a/services/auto-routing-benchmark/src/profiles.test.ts b/services/auto-routing-benchmark/src/profiles.test.ts index 70989de28a2..cf99c934cc6 100644 --- a/services/auto-routing-benchmark/src/profiles.test.ts +++ b/services/auto-routing-benchmark/src/profiles.test.ts @@ -22,6 +22,7 @@ type EventRow = { type PendingStmt = | { kind: 'event'; values: Omit; guarded: boolean } | { kind: 'claim_user_requested'; entries: string[] } + | { kind: 'quota_guard' } | { kind: 'profile'; values: ProfileRow; @@ -277,6 +278,11 @@ vi.mock('drizzle-orm/d1', () => { // against real SQLite in profiles-sql.test.ts. return; } + if (stmt.kind === 'quota_guard') { + // Not simulated: the in-batch quota check runs against real SQLite in + // profile-quota-sql.test.ts. + return; + } if (stmt.kind === 'event') { if (stmt.guarded && hasActiveCurrentProfile(stmt.values)) { // INSERT...SELECT...WHERE NOT EXISTS → zero rows. diff --git a/services/auto-routing-benchmark/src/profiles.ts b/services/auto-routing-benchmark/src/profiles.ts index acf4ad347fe..87deb3bb6c1 100644 --- a/services/auto-routing-benchmark/src/profiles.ts +++ b/services/auto-routing-benchmark/src/profiles.ts @@ -14,7 +14,7 @@ import { } from '@kilocode/auto-routing-contracts'; import type { BatchItem } from 'drizzle-orm/batch'; import { and, asc, eq, gte, inArray, sql } from 'drizzle-orm'; -import { drizzle } from 'drizzle-orm/d1'; +import { drizzle, type DrizzleD1Database } from 'drizzle-orm/d1'; import { benchmarkProfiles, profileRequestEvents } from './db-schema'; import { variantFromStorage, variantToStorage } from './reasoning-effort'; import { computeEngineIdentity } from './run'; @@ -402,6 +402,69 @@ export function chargedEventInsertStatement( ); } +type ProfileRequestEventRow = typeof profileRequestEvents.$inferSelect; + +type AdmissionQuotaScope = { + ownerType: ProfileOwnerType; + ownerId: string; + router: AutoRouter; + windowStartIso: string; +}; + +/** + * Last statement of a charged admission batch. SQLite has no RAISE outside triggers, so + * it fails the batch with a NOT NULL violation when the batch's own events pushed the + * rolling count over the limit; D1 then rolls back every write of that batch. Writer + * serialization makes concurrent admissions count each other's committed events. + */ +function admissionQuotaGuardStatement(orm: DrizzleD1Database, scope: AdmissionQuotaScope) { + return Object.assign( + orm.insert(profileRequestEvents).select(sql` + SELECT NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + WHERE ( + SELECT COUNT(*) FROM ${profileRequestEvents} + WHERE ${profileRequestEvents.owner_type} = ${scope.ownerType} + AND ${profileRequestEvents.owner_id} = ${scope.ownerId} + AND ${profileRequestEvents.router} = ${scope.router} + AND ${profileRequestEvents.admitted_at} >= ${scope.windowStartIso} + ) > ${PROFILE_ADMISSION_LIMIT} + `), + { __stmt: { kind: 'quota_guard' as const } } + ); +} + +function recentAdmissionEvents( + orm: DrizzleD1Database, + scope: AdmissionQuotaScope +): Promise { + return orm + .select() + .from(profileRequestEvents) + .where( + and( + eq(profileRequestEvents.owner_type, scope.ownerType), + eq(profileRequestEvents.owner_id, scope.ownerId), + eq(profileRequestEvents.router, scope.router), + gte(profileRequestEvents.admitted_at, scope.windowStartIso) + ) + ) + .orderBy(asc(profileRequestEvents.admitted_at)); +} + +function profileQuotaExceeded( + recentEvents: ReadonlyArray<{ admitted_at: string }>, + now: Date +): ProfileQuotaExceededError { + const retryAt = computeQuotaRetryAt( + recentEvents[0]?.admitted_at ?? now.toISOString(), + now.getTime() + ); + return new ProfileQuotaExceededError({ + error: `Profile benchmark request limit reached. New benchmarks can be requested after ${retryAt}.`, + retryAt, + }); +} + export type RegisterProfilesInput = { ownerType: ProfileOwnerType; ownerId: string; @@ -467,26 +530,15 @@ export async function registerProfiles( } const orm = drizzle(db); - const recentEvents = await orm - .select() - .from(profileRequestEvents) - .where( - and( - eq(profileRequestEvents.owner_type, input.ownerType), - eq(profileRequestEvents.owner_id, input.ownerId), - eq(profileRequestEvents.router, router), - gte(profileRequestEvents.admitted_at, windowStartIso) - ) - ) - .orderBy(asc(profileRequestEvents.admitted_at)); - + const quotaScope: AdmissionQuotaScope = { + ownerType: input.ownerType, + ownerId: input.ownerId, + router, + windowStartIso, + }; + const recentEvents = await recentAdmissionEvents(orm, quotaScope); if (recentEvents.length + chargedModels.size > PROFILE_ADMISSION_LIMIT) { - const oldest = recentEvents[0]?.admitted_at ?? nowIso; - const retryAt = computeQuotaRetryAt(oldest, now.getTime()); - throw new ProfileQuotaExceededError({ - error: `Profile benchmark request limit reached. New benchmarks can be requested after ${retryAt}.`, - retryAt, - }); + throw profileQuotaExceeded(recentEvents, now); } const admissions = [...chargedAdmissions, ...freeAdmissions]; @@ -525,7 +577,20 @@ export async function registerProfiles( claimUserRequestedStatement(orm, input.pairs, current, nowIso) as BatchItem<'sqlite'> ); - await orm.batch(stmts as [BatchItem<'sqlite'>, ...BatchItem<'sqlite'>[]]); + if (chargedModels.size > 0) { + stmts.push(admissionQuotaGuardStatement(orm, quotaScope) as BatchItem<'sqlite'>); + } + + try { + await orm.batch(stmts as [BatchItem<'sqlite'>, ...BatchItem<'sqlite'>[]]); + } catch (error) { + if (chargedModels.size === 0) throw error; + const committedEvents = await recentAdmissionEvents(orm, quotaScope); + if (committedEvents.length + chargedModels.size > PROFILE_ADMISSION_LIMIT) { + throw profileQuotaExceeded(committedEvents, now); + } + throw error; + } } // Derive reported statuses for admitted entries from post-batch rows so a diff --git a/services/auto-routing/src/custom-router-decision.ts b/services/auto-routing/src/custom-router-decision.ts index d451dd012ce..7bd59da02f5 100644 --- a/services/auto-routing/src/custom-router-decision.ts +++ b/services/auto-routing/src/custom-router-decision.ts @@ -21,6 +21,7 @@ import { CustomRouterError } from './caller-auth'; import type { CustomRouterOwner } from './custom-router-store'; import { customRouterDecisionScope } from './decision-cache'; import type { DecisionRun, DecisionTarget } from './decide'; +import { contextProvablyTooSmall, satisfiesRequiredModalities } from './decision-engine'; import { getModelCapabilities, type ModelCapabilities } from './model-capabilities'; const CUSTOM_ROUTER_TABLE_CACHE_TTL_MS = 60_000; @@ -175,7 +176,7 @@ function fallbackVariant(entry: CustomRouterEntry): string | null { } // A custom router never routes outside its pool: a null decision takes the default entry, then -// the first entry this client can serve (ADR 0005). +// the first entry this client can serve that fits the request (ADR 0005). export function customRouterDecision( routing: CustomRouting, run: DecisionRun @@ -191,7 +192,7 @@ export function customRouterDecision( const fallback = [ ...entries.filter(entry => customRouterEntryKey(entry) === defaultEntryKey), ...entries, - ].find(entry => routing.isServable(entry)); + ].find(entry => routing.isServable(entry) && fitsRequest(entry, run)); if (!fallback) { throw new CustomRouterError( 'custom_router_unavailable', @@ -215,6 +216,18 @@ export function customRouterDecision( }; } +// Same capability rules as the decision engine, without its largest-context rescue: a +// fallback entry that is inactive, lacks a required modality, or is provably too small fails. +function fitsRequest(entry: CustomRouterEntry, run: DecisionRun): boolean { + const caps = run.capabilities.get(customRouterCandidateModel(entry)); + if (caps?.isActive === false) return false; + if (run.constraints === undefined) return true; + return ( + satisfiesRequiredModalities(caps, run.constraints) && + !contextProvablyTooSmall(caps, run.constraints) + ); +} + export function publicCustomRouterDecision(decision: AutoRoutingDecision): CustomRouterDecision { const entry = decision.entry ?? parseCustomRouterCandidateModel(decision.model); if (!entry) throw new Error(`custom router decision ${decision.model} names no pool entry`); diff --git a/services/auto-routing/src/custom-router.test.ts b/services/auto-routing/src/custom-router.test.ts index 5538b4d5b51..2e5af349645 100644 --- a/services/auto-routing/src/custom-router.test.ts +++ b/services/auto-routing/src/custom-router.test.ts @@ -522,6 +522,60 @@ describe('public custom router decide', () => { }); }); + const imageMessages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Fix the layout in this screenshot.' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }, + ], + }, + ]; + + it('falls back past a default entry that lacks a required modality', async () => { + findCustomRouter.mockResolvedValue( + customRouter({ entries: [ollama, sonnet], defaultEntryKey: 'client:ollama/qwen3:32b' }) + ); + classifyNormalizedInput.mockRejectedValue(new Error('classifier down')); + const { decision } = await decisionOf( + await post(CUSTOM_ROUTER_DECIDE_PATH, decideBody({ messages: imageMessages })) + ); + expect(decision).toMatchObject({ model: SONNET, fallback: true }); + }); + + it('fails with custom_router_unavailable when the classifier fails and no entry fits', async () => { + findCustomRouter.mockResolvedValue( + customRouter({ entries: [ollama, lmstudio], defaultEntryKey: 'client:ollama/qwen3:32b' }) + ); + classifyNormalizedInput.mockRejectedValue(new Error('classifier down')); + const res = await post( + CUSTOM_ROUTER_DECIDE_PATH, + decideBody({ + messages: imageMessages, + clientEntries: [ + { provider: 'ollama', model: 'qwen3:32b' }, + { provider: 'lmstudio', model: 'gemma' }, + ], + }) + ); + expect(res.status).toBe(422); + expect(await errorCode(res)).toBe('custom_router_unavailable'); + }); + + it('accepts a body at the size limit and rejects one byte more', async () => { + const limit = 32_000_000; + const padded = (bytes: number) => { + const unpadded = JSON.stringify({ ...decideBody(), padding: '' }).length; + return { ...decideBody(), padding: 'x'.repeat(bytes - unpadded) }; + }; + await decisionOf(await post(CUSTOM_ROUTER_DECIDE_PATH, padded(limit))); + const tooLarge = await post(CUSTOM_ROUTER_DECIDE_PATH, padded(limit + 1)); + expect(tooLarge.status).toBe(400); + expect(await tooLarge.json()).toEqual({ + error: { code: 'invalid_request', message: 'Request body is too large' }, + }); + }); + it('drops gateway entries the organization policy denies the member', async () => { gatewayDeniedModelIds = [SONNET]; const { decision } = await decisionOf( diff --git a/services/auto-routing/src/decide.ts b/services/auto-routing/src/decide.ts index d23ba525291..a33f32da4ed 100644 --- a/services/auto-routing/src/decide.ts +++ b/services/auto-routing/src/decide.ts @@ -411,6 +411,8 @@ export type DecisionTarget = { export type DecisionRun = { response: AutoRoutingDecisionResponse; routingTable: RoutingTable | null; + capabilities: ModelCapabilitiesMap; + constraints: RoutingConstraints | undefined; }; function platformTarget( @@ -748,6 +750,8 @@ export async function runDecision( return { response: { cost: 0, decision: outcome.decision, classifierResult: null }, routingTable, + capabilities, + constraints, }; } if (outcome.kind === 'cache_hit') { @@ -815,7 +819,12 @@ export async function runDecision( shadow ); } - return { response: decisionResponse(cached, payload.input, decision), routingTable }; + return { + response: decisionResponse(cached, payload.input, decision), + routingTable, + capabilities, + constraints, + }; } try { @@ -878,6 +887,8 @@ export async function runDecision( return { response: decisionResponse(classifier.classification, payload.input, decision), routingTable, + capabilities, + constraints, }; } catch (error) { recordDecision( @@ -890,7 +901,7 @@ export async function runDecision( sticky, speculative ? 'speculative' : 'sync' ); - return { response: emptyDecisionResponse(), routingTable }; + return { response: emptyDecisionResponse(), routingTable, capabilities, constraints }; } } diff --git a/services/auto-routing/src/public-routes.ts b/services/auto-routing/src/public-routes.ts index 5545984bdc6..6a3bbf16b9f 100644 --- a/services/auto-routing/src/public-routes.ts +++ b/services/auto-routing/src/public-routes.ts @@ -22,6 +22,7 @@ import { import type { OrganizationRole } from '@kilocode/db'; import { formatError, getCachedSecret } from '@kilocode/worker-utils'; import { canSpendPaidUsage } from '@kilocode/worker-utils/account-access'; +import { readBoundedStream } from '@kilocode/worker-utils/bounded-stream-reader'; import type { Context, Handler } from 'hono'; import type * as z from 'zod'; import { customRouterBenchmarkContext, forwardClientEval } from './benchmark-origin'; @@ -64,10 +65,21 @@ function publicHandler(handle: (c: Context) => Promise): Hand }; } -async function readJson(c: Context): Promise<{ text: string; json: unknown }> { - const text = await c.req.text(); +// A decide request carries the whole conversation, inline images included. The limit matches +// the largest request an upstream model provider accepts (Anthropic Messages, 32 MB). +const MAX_PUBLIC_REQUEST_BODY_BYTES = 32 * 1000 * 1000; + +async function readJson(c: Context): Promise<{ byteLength: number; json: unknown }> { + const body = c.req.raw.body; + const read = body + ? await readBoundedStream(body as ReadableStream, MAX_PUBLIC_REQUEST_BODY_BYTES) + : { ok: true as const, bytes: new Uint8Array() }; + if (!read.ok) throw new CustomRouterError('invalid_request', 'Request body is too large'); try { - return { text, json: JSON.parse(text) }; + return { + byteLength: read.bytes.byteLength, + json: JSON.parse(new TextDecoder().decode(read.bytes)), + }; } catch { throw new CustomRouterError('invalid_request', 'Invalid JSON body'); } @@ -92,7 +104,7 @@ function requirePaidUsageBalance(balanceUsd: number): void { export const publicDecideHandler = publicHandler(async c => { const caller = await authenticateCaller(c.env, c.req.raw.headers); - const { text, json } = await readJson(c); + const { byteLength, json } = await readJson(c); const request = parseBody(CustomRouterDecideRequestSchema, json); const reference = parseCustomRouterModelId(request.model); if (!reference) throw new CustomRouterError('invalid_request', 'Not a custom router model ID'); @@ -160,7 +172,7 @@ export const publicDecideHandler = publicHandler(async c => { clientRequestId: request.clientRequestId ?? null, mode: request.mode ?? null, userAgent: c.req.header('user-agent') ?? null, - bodyBytes: new TextEncoder().encode(text).byteLength, + bodyBytes: byteLength, customRouter: reference, clientEntries: request.clientEntries, }; From 6bb8c745711b9e587b4547b52c4c3764f966327c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Mon, 19 Oct 2026 14:02:34 +0200 Subject: [PATCH 2/2] fix(auto-routing-benchmark): drop a client eval result whose pending usage is unreadable --- .../src/client-eval.test.ts | 16 +++++++++++ .../auto-routing-benchmark/src/client-eval.ts | 27 ++++++++++++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/services/auto-routing-benchmark/src/client-eval.test.ts b/services/auto-routing-benchmark/src/client-eval.test.ts index 53cc1663d69..90be1aab0b6 100644 --- a/services/auto-routing-benchmark/src/client-eval.test.ts +++ b/services/auto-routing-benchmark/src/client-eval.test.ts @@ -463,6 +463,22 @@ describe('client eval (real SQLite)', () => { sqlite.prepare('SELECT pending_grader_usage AS pending FROM client_case_results').get() ).toEqual({ pending: null }); }); + + it('drops a result whose pending grader usage is unreadable so a retry grades it again', async () => { + const judged = { caseId: judgedCase.id }; + await gradeClientEvalResult(db, CONFIG, upload(judged), deps); + sqlite.exec(`UPDATE client_case_results SET pending_grader_usage = '{not json'`); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(gradeClientEvalResult(db, CONFIG, upload(judged), deps)).rejects.toMatchObject({ + code: 'internal_error', + }); + expect(sqlite.prepare('SELECT COUNT(*) AS n FROM client_case_results').get()).toEqual({ n: 0 }); + expect(await gradeClientEvalResult(db, CONFIG, upload(judged), deps)).toEqual({ score: 0.75 }); + expect(judge).toHaveBeenCalledTimes(2); + expect(logged).toHaveBeenCalledOnce(); + logged.mockRestore(); + }); }); describe('clientEvalCase', () => { diff --git a/services/auto-routing-benchmark/src/client-eval.ts b/services/auto-routing-benchmark/src/client-eval.ts index 1ecc60d6357..4204bfd3a29 100644 --- a/services/auto-routing-benchmark/src/client-eval.ts +++ b/services/auto-routing-benchmark/src/client-eval.ts @@ -213,9 +213,30 @@ export async function gradeClientEvalResult( const stored = await getClientCase(db, caseKey); if (stored?.ran) { if (stored.pendingGraderUsage !== null) { - const usage = UsageRecordRequestSchema.parse(JSON.parse(stored.pendingGraderUsage)); - if (!(await recordGraderUsage(deps, usage))) throw graderUsageNotRecorded(); - await clearPendingGraderUsage(db, { ...caseKey, revision: stored.revision }); + const storedKey = { ...caseKey, revision: stored.revision }; + let pending: unknown; + try { + pending = JSON.parse(stored.pendingGraderUsage); + } catch { + pending = null; + } + const usage = UsageRecordRequestSchema.safeParse(pending); + if (!usage.success) { + // An unreadable record cannot be billed; drop the result so a retry grades it again. + console.error( + JSON.stringify({ + event: 'client_eval_pending_usage_invalid', + profileId: profile.id, + caseId: request.caseId, + rep: request.rep, + }) + ); + await deleteClientCaseUpload(db, storedKey); + await refreshClientProfileRollup(db, profile, plannedResults, nowIso); + throw graderUsageNotRecorded(); + } + if (!(await recordGraderUsage(deps, usage.data))) throw graderUsageNotRecorded(); + await clearPendingGraderUsage(db, storedKey); } await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: stored.score };