diff --git a/packages/auto-routing-contracts/src/custom-router.test.ts b/packages/auto-routing-contracts/src/custom-router.test.ts index 888cbd6e590..a60f929e15b 100644 --- a/packages/auto-routing-contracts/src/custom-router.test.ts +++ b/packages/auto-routing-contracts/src/custom-router.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { AUTO_DECISION_HEADER, + AUTO_DECISION_TTL_MS, CUSTOM_ROUTER_ERROR_HTTP_STATUS, ClientCustomRouterEntrySchema, ClientEvalPlanResponseSchema, @@ -279,6 +280,23 @@ describe('public decide API', () => { ).toBe(true); }); + it('keeps the classification on a fallback decision', () => { + const decision = { + entry: { kind: 'gateway', model: 'anthropic/claude-sonnet-4' }, + model: 'anthropic/claude-sonnet-4', + variant: null, + stale: false, + fallback: true, + sticky: false, + taskType: 'debugging', + subtaskType: 'bug_fixing', + tableVersion: null, + }; + expect( + CustomRouterDecideResponseSchema.parse({ decision, decisionId: 'v1.abc.def' }).decision + ).toMatchObject({ fallback: true, taskType: 'debugging', subtaskType: 'bug_fixing' }); + }); + it('maps every error code to a status', () => { for (const code of CustomRouterErrorCodeSchema.options) { expect(CUSTOM_ROUTER_ERROR_HTTP_STATUS[code]).toBeGreaterThanOrEqual(400); @@ -340,6 +358,25 @@ describe('auto decision tokens', () => { }); }); + it('rejects a token whose expiry exceeds the decision lifetime', async () => { + const nowMs = 1_000; + const longLived = await signAutoDecision( + { ...decisionPayload, expiresAt: nowMs + AUTO_DECISION_TTL_MS + 60_000 }, + DECISION_SECRET + ); + await expect(verifyAutoDecision(longLived, DECISION_SECRET, nowMs)).resolves.toEqual({ + ok: false, + reason: 'malformed', + }); + const fullLifetime = await signAutoDecision( + { ...decisionPayload, expiresAt: nowMs + AUTO_DECISION_TTL_MS }, + DECISION_SECRET + ); + await expect(verifyAutoDecision(fullLifetime, DECISION_SECRET, nowMs)).resolves.toMatchObject({ + ok: true, + }); + }); + it('rejects expired and malformed tokens', async () => { const expired = await signAutoDecision( { ...decisionPayload, expiresAt: 1_000 }, diff --git a/packages/auto-routing-contracts/src/custom-router.ts b/packages/auto-routing-contracts/src/custom-router.ts index d1c3b564c28..6e4a7263fdc 100644 --- a/packages/auto-routing-contracts/src/custom-router.ts +++ b/packages/auto-routing-contracts/src/custom-router.ts @@ -271,7 +271,8 @@ export const CustomRouterDecisionSchema = z model: z.string().min(1), variant: z.string().trim().min(1).nullable(), stale: z.boolean(), - // True when the classifier produced no decision and the default or first servable entry was used. + // True when no routing decision named a servable entry, so the default or first servable + // entry was used. taskType and subtaskType still report the classification, if any. fallback: z.boolean(), sticky: z.boolean(), taskType: ClassifierTaskTypeSchema.nullable(), @@ -296,6 +297,7 @@ export type CustomRouterDecideResponse = z.infer nowMs + AUTO_DECISION_TTL_MS + AUTO_DECISION_CLOCK_SKEW_MS) { + return { ok: false, reason: 'malformed' }; + } return { ok: true, payload: payload.data }; } diff --git a/services/auto-routing-benchmark/migrations/0026_complete_triton.sql b/services/auto-routing-benchmark/migrations/0026_complete_triton.sql new file mode 100644 index 00000000000..fec71921069 --- /dev/null +++ b/services/auto-routing-benchmark/migrations/0026_complete_triton.sql @@ -0,0 +1 @@ +ALTER TABLE `client_case_results` ADD `pending_grader_usage` text; \ No newline at end of file diff --git a/services/auto-routing-benchmark/migrations/meta/0026_snapshot.json b/services/auto-routing-benchmark/migrations/meta/0026_snapshot.json new file mode 100644 index 00000000000..8bada756585 --- /dev/null +++ b/services/auto-routing-benchmark/migrations/meta/0026_snapshot.json @@ -0,0 +1,2225 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "12022eb4-dff2-4d67-9215-bc0fded7e90f", + "prevId": "90c397cc-b7c4-4274-ba17-c3f578c3b9b2", + "tables": { + "benchmark_config": { + "name": "benchmark_config", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy_noise_band": { + "name": "accuracy_noise_band", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "frontier_min_accuracy_delta": { + "name": "frontier_min_accuracy_delta", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "production_token_profile": { + "name": "production_token_profile", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "switch_horizon_turns": { + "name": "switch_horizon_turns", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "low_confidence_threshold": { + "name": "low_confidence_threshold", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_concurrency": { + "name": "max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_max_concurrency": { + "name": "user_max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 100 + }, + "benchmark_user_id": { + "name": "benchmark_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "benchmark_org_id": { + "name": "benchmark_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "classifier_repetitions": { + "name": "classifier_repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "decider_repetitions": { + "name": "decider_repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "classifier_max_p95_latency_ms": { + "name": "classifier_max_p95_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "policy_overrides": { + "name": "policy_overrides", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "policy_computed": { + "name": "policy_computed", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "benchmark_config_router_unique": { + "name": "benchmark_config_router_unique", + "columns": [ + "router" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_profiles": { + "name": "benchmark_profiles", + "columns": { + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invalidated_at": { + "name": "invalidated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_at": { + "name": "requested_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "platform_requested": { + "name": "platform_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "user_requested": { + "name": "user_requested", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "benchmark_profiles_router_model_variant_engine_identity_repetitions_pk": { + "columns": [ + "router", + "model", + "variant", + "engine_identity", + "repetitions" + ], + "name": "benchmark_profiles_router_model_variant_engine_identity_repetitions_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_runs": { + "name": "benchmark_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy_noise_band": { + "name": "accuracy_noise_band", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "frontier_min_accuracy_delta": { + "name": "frontier_min_accuracy_delta", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "max_concurrency": { + "name": "max_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "benchmark_user_id": { + "name": "benchmark_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "benchmark_org_id": { + "name": "benchmark_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "classifier_max_p95_latency_ms": { + "name": "classifier_max_p95_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'platform'" + } + }, + "indexes": { + "UQ_benchmark_runs_one_running_per_router_kind_purpose": { + "name": "UQ_benchmark_runs_one_running_per_router_kind_purpose", + "columns": [ + "router", + "kind", + "purpose" + ], + "isUnique": true, + "where": "\"benchmark_runs\".\"status\" = 'running'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "benchmark_subscriptions": { + "name": "benchmark_subscriptions", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot": { + "name": "snapshot", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "benchmark_subscriptions_router_owner_type_owner_id_pk": { + "columns": [ + "router", + "owner_type", + "owner_id" + ], + "name": "benchmark_subscriptions_router_owner_type_owner_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "case_results": { + "name": "case_results", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fallback_reason": { + "name": "fallback_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "route_hit": { + "name": "route_hit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retried": { + "name": "retried", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_prefix": { + "name": "output_prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "event_count": { + "name": "event_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_event_types": { + "name": "last_event_types", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "timed_out": { + "name": "timed_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "prompt_revision": { + "name": "prompt_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "case_results_run_id_model_variant_case_id_rep_pk": { + "columns": [ + "run_id", + "model", + "variant", + "case_id", + "rep" + ], + "name": "case_results_run_id_model_variant_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "client_case_results": { + "name": "client_case_results", + "columns": { + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timed_out": { + "name": "timed_out", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "score": { + "name": "score", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "grader_cost_usd": { + "name": "grader_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded_at": { + "name": "uploaded_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "pending_grader_usage": { + "name": "pending_grader_usage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "client_case_results_profile_id_client_profiles_id_fk": { + "name": "client_case_results_profile_id_client_profiles_id_fk", + "tableFrom": "client_case_results", + "tableTo": "client_profiles", + "columnsFrom": [ + "profile_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "client_case_results_profile_id_case_id_rep_pk": { + "columns": [ + "profile_id", + "case_id", + "rep" + ], + "name": "client_case_results_profile_id_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "client_profiles": { + "name": "client_profiles", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entry_kind": { + "name": "entry_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "results": { + "name": "results", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errors": { + "name": "errors", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "timeouts": { + "name": "timeouts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "p50_latency_ms": { + "name": "p50_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "p95_latency_ms": { + "name": "p95_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "route_summaries": { + "name": "route_summaries", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "upload_seq": { + "name": "upload_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "UQ_client_profiles_identity": { + "name": "UQ_client_profiles_identity", + "columns": [ + "owner_type", + "owner_id", + "router", + "entry_kind", + "provider", + "model", + "variant", + "engine_identity", + "repetitions" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_classifier_models": { + "name": "config_classifier_models", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "config_classifier_models_router_model_pk": { + "columns": [ + "router", + "model" + ], + "name": "config_classifier_models_router_model_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config_decider_models": { + "name": "config_decider_models", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "config_decider_models_router_model_pk": { + "columns": [ + "router", + "model" + ], + "name": "config_decider_models_router_model_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_summaries": { + "name": "model_summaries", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avg_latency_ms": { + "name": "avg_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "p50_latency_ms": { + "name": "p50_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cases": { + "name": "cases", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "errors": { + "name": "errors", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "p95_latency_ms": { + "name": "p95_latency_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timeouts": { + "name": "timeouts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "route_accuracy": { + "name": "route_accuracy", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "carried": { + "name": "carried", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "model_summaries_run_id_model_variant_route_key_pk": { + "columns": [ + "run_id", + "model", + "variant", + "route_key" + ], + "name": "model_summaries_run_id_model_variant_route_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pending_decider_outputs": { + "name": "pending_decider_outputs", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt_revision": { + "name": "prompt_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": {}, + "foreignKeys": { + "pending_decider_outputs_run_id_benchmark_runs_id_fk": { + "name": "pending_decider_outputs_run_id_benchmark_runs_id_fk", + "tableFrom": "pending_decider_outputs", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "pending_decider_outputs_run_id_model_variant_case_id_rep_pk": { + "columns": [ + "run_id", + "model", + "variant", + "case_id", + "rep" + ], + "name": "pending_decider_outputs_run_id_model_variant_case_id_rep_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "profile_remeasures": { + "name": "profile_remeasures", + "columns": { + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_at": { + "name": "requested_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "IDX_profile_remeasures_run": { + "name": "IDX_profile_remeasures_run", + "columns": [ + "run_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "profile_remeasures_router_model_variant_engine_identity_repetitions_pk": { + "columns": [ + "router", + "model", + "variant", + "engine_identity", + "repetitions" + ], + "name": "profile_remeasures_router_model_variant_engine_identity_repetitions_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "profile_request_events": { + "name": "profile_request_events", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "engine_identity": { + "name": "engine_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repetitions": { + "name": "repetitions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "admitted_at": { + "name": "admitted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "IDX_profile_request_events_owner_router_admitted": { + "name": "IDX_profile_request_events_owner_router_admitted", + "columns": [ + "owner_type", + "owner_id", + "router", + "admitted_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "routing_table_candidates": { + "name": "routing_table_candidates", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "route_key": { + "name": "route_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rank": { + "name": "rank", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accuracy": { + "name": "accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cases": { + "name": "cases", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avg_cost_usd": { + "name": "avg_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "meets_threshold": { + "name": "meets_threshold", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "pricing": { + "name": "pricing", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "estimated_request_cost_usd": { + "name": "estimated_request_cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "routing_table_candidates_run_id_route_key_rank_pk": { + "columns": [ + "run_id", + "route_key", + "rank" + ], + "name": "routing_table_candidates_run_id_route_key_rank_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "routing_tables": { + "name": "routing_tables", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "router": { + "name": "router", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'efficient'" + }, + "published_at": { + "name": "published_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "generated_at": { + "name": "generated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_accuracy": { + "name": "min_accuracy", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "switch_cost_factor": { + "name": "switch_cost_factor", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "best_accuracy_switch_threshold": { + "name": "best_accuracy_switch_threshold", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0.05 + }, + "production_token_profile": { + "name": "production_token_profile", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "switch_horizon_turns": { + "name": "switch_horizon_turns", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "low_confidence_threshold": { + "name": "low_confidence_threshold", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "effort_by_complexity": { + "name": "effort_by_complexity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_lane_failures": { + "name": "run_lane_failures", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "chunk": { + "name": "chunk", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "shard": { + "name": "shard", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "failed_at": { + "name": "failed_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "run_lane_failures_run_id_model_variant_rep_chunk_shard_pk": { + "columns": [ + "run_id", + "model", + "variant", + "rep", + "chunk", + "shard" + ], + "name": "run_lane_failures_run_id_model_variant_rep_chunk_shard_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_models": { + "name": "run_models", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "enqueued": { + "name": "enqueued", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "run_models_run_id_model_variant_pk": { + "columns": [ + "run_id", + "model", + "variant" + ], + "name": "run_models_run_id_model_variant_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_chunks": { + "name": "scheduler_chunks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providers": { + "name": "providers", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "chunk": { + "name": "chunk", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "case_ids": { + "name": "case_ids", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "slot": { + "name": "slot", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "available_at": { + "name": "available_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "IX_scheduler_chunks_pending": { + "name": "IX_scheduler_chunks_pending", + "columns": [ + "status", + "available_at" + ], + "isUnique": false + }, + "IX_scheduler_chunks_run": { + "name": "IX_scheduler_chunks_run", + "columns": [ + "run_id" + ], + "isUnique": false + }, + "IX_scheduler_chunks_lane": { + "name": "IX_scheduler_chunks_lane", + "columns": [ + "status", + "run_id", + "model", + "variant", + "rep", + "available_at", + "chunk" + ], + "isUnique": false + }, + "UQ_scheduler_chunks_live_slot": { + "name": "UQ_scheduler_chunks_live_slot", + "columns": [ + "slot" + ], + "isUnique": true, + "where": "\"scheduler_chunks\".\"status\" IN ('queued', 'active')" + } + }, + "foreignKeys": { + "scheduler_chunks_run_id_benchmark_runs_id_fk": { + "name": "scheduler_chunks_run_id_benchmark_runs_id_fk", + "tableFrom": "scheduler_chunks", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_cooldowns": { + "name": "scheduler_cooldowns", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "until": { + "name": "until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "scheduler_cooldowns_scope_key_pk": { + "columns": [ + "scope", + "key" + ], + "name": "scheduler_cooldowns_scope_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_dispatch_lock": { + "name": "scheduler_dispatch_lock", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_executions": { + "name": "scheduler_executions", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "variant": { + "name": "variant", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rep": { + "name": "rep", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "case_id": { + "name": "case_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "prompt_revision": { + "name": "prompt_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + } + }, + "indexes": {}, + "foreignKeys": { + "scheduler_executions_run_id_benchmark_runs_id_fk": { + "name": "scheduler_executions_run_id_benchmark_runs_id_fk", + "tableFrom": "scheduler_executions", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "scheduler_executions_run_id_model_variant_rep_case_id_pk": { + "columns": [ + "run_id", + "model", + "variant", + "rep", + "case_id" + ], + "name": "scheduler_executions_run_id_model_variant_rep_case_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_runs": { + "name": "scheduler_runs", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "scheduler_runs_run_id_benchmark_runs_id_fk": { + "name": "scheduler_runs_run_id_benchmark_runs_id_fk", + "tableFrom": "scheduler_runs", + "tableTo": "benchmark_runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "scheduler_settings": { + "name": "scheduler_settings", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "slots": { + "name": "slots", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_concurrency": { + "name": "model_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_concurrency": { + "name": "provider_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "unknown_provider_concurrency": { + "name": "unknown_provider_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/services/auto-routing-benchmark/migrations/meta/_journal.json b/services/auto-routing-benchmark/migrations/meta/_journal.json index 00759578ecd..bd44411a3c6 100644 --- a/services/auto-routing-benchmark/migrations/meta/_journal.json +++ b/services/auto-routing-benchmark/migrations/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1792408783184, "tag": "0025_black_black_tom", "breakpoints": true + }, + { + "idx": 26, + "version": "6", + "when": 1792414121558, + "tag": "0026_complete_triton", + "breakpoints": true } ] } \ No newline at end of file diff --git a/services/auto-routing-benchmark/src/client-eval.test.ts b/services/auto-routing-benchmark/src/client-eval.test.ts index 337c1eb3509..90be1aab0b6 100644 --- a/services/auto-routing-benchmark/src/client-eval.test.ts +++ b/services/auto-routing-benchmark/src/client-eval.test.ts @@ -444,6 +444,41 @@ describe('client eval (real SQLite)', () => { expect(second?.id).not.toBe(first?.id); expect(sqlite.prepare('SELECT COUNT(*) AS n FROM client_case_results').get()).toEqual({ n: 1 }); }); + + it('resends the stored grader usage on retry when removing an unbilled upload fails', async () => { + const judged = { caseId: judgedCase.id }; + sqlite.exec( + "CREATE TRIGGER keep_case_results BEFORE DELETE ON client_case_results BEGIN SELECT RAISE(ABORT, 'delete failed'); END" + ); + writeUsage.mockResolvedValueOnce({ kind: 'unavailable', reason: 'http_error' }); + await expect(gradeClientEvalResult(db, CONFIG, upload(judged), deps)).rejects.toThrow(); + + expect(await gradeClientEvalResult(db, CONFIG, upload(judged), deps)).toEqual({ score: 0.75 }); + expect(await gradeClientEvalResult(db, CONFIG, upload(judged), deps)).toEqual({ score: 0.75 }); + expect(writeUsage).toHaveBeenCalledTimes(2); + const [first, resent] = writeUsage.mock.calls.map(([usage]) => usage); + expect(resent).toEqual(first); + expect(judge).toHaveBeenCalledOnce(); + expect( + sqlite.prepare('SELECT pending_grader_usage AS pending FROM client_case_results').get() + ).toEqual({ pending: null }); + }); + + it('drops a result whose pending grader usage is unreadable so a retry grades it again', async () => { + const judged = { caseId: judgedCase.id }; + await gradeClientEvalResult(db, CONFIG, upload(judged), deps); + sqlite.exec(`UPDATE client_case_results SET pending_grader_usage = '{not json'`); + const logged = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await expect(gradeClientEvalResult(db, CONFIG, upload(judged), deps)).rejects.toMatchObject({ + code: 'internal_error', + }); + expect(sqlite.prepare('SELECT COUNT(*) AS n FROM client_case_results').get()).toEqual({ n: 0 }); + expect(await gradeClientEvalResult(db, CONFIG, upload(judged), deps)).toEqual({ score: 0.75 }); + expect(judge).toHaveBeenCalledTimes(2); + expect(logged).toHaveBeenCalledOnce(); + logged.mockRestore(); + }); }); describe('clientEvalCase', () => { diff --git a/services/auto-routing-benchmark/src/client-eval.ts b/services/auto-routing-benchmark/src/client-eval.ts index 9b5a5886a8c..4204bfd3a29 100644 --- a/services/auto-routing-benchmark/src/client-eval.ts +++ b/services/auto-routing-benchmark/src/client-eval.ts @@ -22,13 +22,14 @@ import { type InternalClientEvalPlanRequest, type InternalClientEvalResultRequest, } from '@kilocode/auto-routing-contracts'; -import type { UsageRecordRequest } from '@kilocode/usage-contracts'; +import { UsageRecordRequestSchema, type UsageRecordRequest } from '@kilocode/usage-contracts'; import { formatError } from '@kilocode/worker-utils'; import type { UsageWriteOutcome } from '@kilocode/worker-utils/usage-writer-client'; import type { Hono } from 'hono'; import type * as z from 'zod'; import { DECIDER_CLI_TIMEOUT_MS, deciderCliPrompt } from './cli-runner'; import { + clearPendingGraderUsage, clientCaseKey, clientPairKey, deleteClientCaseUpload, @@ -211,6 +212,32 @@ export async function gradeClientEvalResult( const plannedResults = deciderCasesForRouter(request.baseRouter).length * current.repetitions; const stored = await getClientCase(db, caseKey); if (stored?.ran) { + if (stored.pendingGraderUsage !== null) { + const storedKey = { ...caseKey, revision: stored.revision }; + let pending: unknown; + try { + pending = JSON.parse(stored.pendingGraderUsage); + } catch { + pending = null; + } + const usage = UsageRecordRequestSchema.safeParse(pending); + if (!usage.success) { + // An unreadable record cannot be billed; drop the result so a retry grades it again. + console.error( + JSON.stringify({ + event: 'client_eval_pending_usage_invalid', + profileId: profile.id, + caseId: request.caseId, + rep: request.rep, + }) + ); + await deleteClientCaseUpload(db, storedKey); + await refreshClientProfileRollup(db, profile, plannedResults, nowIso); + throw graderUsageNotRecorded(); + } + if (!(await recordGraderUsage(deps, usage.data))) throw graderUsageNotRecorded(); + await clearPendingGraderUsage(db, storedKey); + } await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: stored.score }; } @@ -227,6 +254,29 @@ export async function gradeClientEvalResult( throw new ClientEvalError('internal_error', 'The grader is unavailable; retry the upload.'); } + const usage = + grade.costUsd > 0 + ? await buildGraderUsage( + request.billing, + [ + request.billing.userId, + request.billing.organizationId, + request.ownerType, + request.ownerId, + request.baseRouter, + entryKey, + request.variant, + current.engineIdentity, + current.repetitions, + request.caseId, + request.rep, + ], + grade.costUsd, + nowIso + ).catch(() => { + throw graderUsageNotRecorded(); + }) + : null; const revision = await saveClientCaseResult(db, { profile_id: profile.id, case_id: request.caseId, @@ -240,50 +290,37 @@ export async function gradeClientEvalResult( score: grade.score, grader_cost_usd: grade.costUsd, uploaded_at: nowIso, + pending_grader_usage: usage === null ? null : JSON.stringify(usage), }); if (revision === null) { await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: (await getClientCase(db, caseKey))?.score ?? grade.score }; } - if (grade.costUsd > 0) { - const billed = await buildGraderUsage( - request.billing, - [ - request.billing.userId, - request.billing.organizationId, - request.ownerType, - request.ownerId, - request.baseRouter, - entryKey, - request.variant, - current.engineIdentity, - current.repetitions, - request.caseId, - request.rep, - ], - grade.costUsd, - nowIso - ) - .then(usage => deps.writeUsage(usage)) - .then( - outcome => outcome.kind !== 'unavailable', - () => false - ); - if (!billed) { + if (usage !== null) { + if (!(await recordGraderUsage(deps, usage))) { await deleteClientCaseUpload(db, { ...caseKey, revision }); await refreshClientProfileRollup(db, profile, plannedResults, nowIso); - throw new ClientEvalError( - 'internal_error', - 'Grader usage was not recorded; retry the upload.' - ); + throw graderUsageNotRecorded(); } + await clearPendingGraderUsage(db, { ...caseKey, revision }); } await refreshClientProfileRollup(db, profile, plannedResults, nowIso); return { score: grade.score }; } +function recordGraderUsage(deps: ClientEvalDeps, usage: UsageRecordRequest): Promise { + return deps.writeUsage(usage).then( + outcome => outcome.kind !== 'unavailable', + () => false + ); +} + +function graderUsageNotRecorded(): ClientEvalError { + return new ClientEvalError('internal_error', 'Grader usage was not recorded; retry the upload.'); +} + async function requireConfig(db: D1Database, router: AutoRouter): Promise { const config = await getBenchmarkConfig(db, router); if (!config) { diff --git a/services/auto-routing-benchmark/src/client-profiles.ts b/services/auto-routing-benchmark/src/client-profiles.ts index 7b8d0b96a4f..79a6bbfe3b6 100644 --- a/services/auto-routing-benchmark/src/client-profiles.ts +++ b/services/auto-routing-benchmark/src/client-profiles.ts @@ -144,12 +144,19 @@ const ranSuccessfully = sql`(${clientCaseResults.exit_code} IS NOT NULL AND ${cl export async function getClientCase( db: D1Database, key: { profileId: string; caseId: string; rep: number } -): Promise<{ score: number; ran: boolean } | null> { +): Promise<{ + score: number; + ran: boolean; + revision: number; + pendingGraderUsage: string | null; +} | null> { const [row] = await drizzle(db) .select({ score: clientCaseResults.score, exitCode: clientCaseResults.exit_code, timedOut: clientCaseResults.timed_out, + revision: clientCaseResults.revision, + pendingGraderUsage: clientCaseResults.pending_grader_usage, }) .from(clientCaseResults) .where( @@ -159,7 +166,14 @@ export async function getClientCase( eq(clientCaseResults.rep, key.rep) ) ); - return row ? { score: row.score, ran: row.exitCode === 0 && !row.timedOut } : null; + return row + ? { + score: row.score, + ran: row.exitCode === 0 && !row.timedOut, + revision: row.revision, + pendingGraderUsage: row.pendingGraderUsage, + } + : null; } /** @@ -194,6 +208,7 @@ export async function saveClientCaseResult( timed_out: row.timed_out, score: row.score, grader_cost_usd: row.grader_cost_usd, + pending_grader_usage: row.pending_grader_usage, uploaded_at: row.uploaded_at, revision: sql`excluded.revision`, }, @@ -208,17 +223,33 @@ export async function saveClientCaseResult( export async function deleteClientCaseUpload( db: D1Database, key: { profileId: string; caseId: string; rep: number; revision: number } +): Promise { + await drizzle(db).delete(clientCaseResults).where(clientCaseUploadWhere(key)); +} + +/** Mark one upload's grader usage as acknowledged by the usage writer. */ +export async function clearPendingGraderUsage( + db: D1Database, + key: { profileId: string; caseId: string; rep: number; revision: number } ): Promise { await drizzle(db) - .delete(clientCaseResults) - .where( - and( - eq(clientCaseResults.profile_id, key.profileId), - eq(clientCaseResults.case_id, key.caseId), - eq(clientCaseResults.rep, key.rep), - eq(clientCaseResults.revision, key.revision) - ) - ); + .update(clientCaseResults) + .set({ pending_grader_usage: null }) + .where(clientCaseUploadWhere(key)); +} + +function clientCaseUploadWhere(key: { + profileId: string; + caseId: string; + rep: number; + revision: number; +}) { + return and( + eq(clientCaseResults.profile_id, key.profileId), + eq(clientCaseResults.case_id, key.caseId), + eq(clientCaseResults.rep, key.rep), + eq(clientCaseResults.revision, key.revision) + ); } /** `${caseId}\0${rep}` keys of cases that ran, per profile id. */ @@ -255,7 +286,7 @@ export function clientCaseKey(caseId: string, rep: number): string { return `${caseId}\0${rep}`; } -type GradedClientCase = Omit; +type GradedClientCase = Omit; async function listGradedClientCases( db: D1Database, diff --git a/services/auto-routing-benchmark/src/db-schema.ts b/services/auto-routing-benchmark/src/db-schema.ts index ff33eb4cb94..73b11355907 100644 --- a/services/auto-routing-benchmark/src/db-schema.ts +++ b/services/auto-routing-benchmark/src/db-schema.ts @@ -554,6 +554,8 @@ export const clientCaseResults = sqliteTable( // This upload's revision from the profile's `upload_seq`, unique within the profile even // across same-millisecond uploads and delete-and-reinsert; the rollup guard compares it. revision: integer('revision').notNull().default(0), + // Grader usage the usage writer has not acknowledged yet; a retried upload resends it. + pending_grader_usage: text('pending_grader_usage'), }, table => [primaryKey({ columns: [table.profile_id, table.case_id, table.rep] })] ); diff --git a/services/auto-routing-benchmark/src/profile-quota-sql.test.ts b/services/auto-routing-benchmark/src/profile-quota-sql.test.ts index 52dd99723ec..6b6fcbddb75 100644 --- a/services/auto-routing-benchmark/src/profile-quota-sql.test.ts +++ b/services/auto-routing-benchmark/src/profile-quota-sql.test.ts @@ -71,4 +71,26 @@ describe('platform-paid admission quota (real SQLite)', () => { n: 0, }); }); + + it('admits only the remaining capacity when concurrent requests race at the limit', async () => { + const filled = models('filled', PROFILE_ADMISSION_LIMIT - 1); + for (let offset = 0; offset < filled.length; offset += MAX_POOL_ENTRIES) { + await admit('efficient', filled.slice(offset, offset + MAX_POOL_ENTRIES)); + } + + const results = await Promise.allSettled([ + admit('efficient', ['race/a']), + admit('efficient', ['race/b']), + ]); + + expect(results.map(result => result.status).toSorted()).toEqual(['fulfilled', 'rejected']); + const rejected = results.find(result => result.status === 'rejected'); + expect(rejected?.reason).toBeInstanceOf(ProfileQuotaExceededError); + expect(sqlite.prepare('SELECT COUNT(*) AS n FROM profile_request_events').get()).toEqual({ + n: PROFILE_ADMISSION_LIMIT, + }); + expect( + sqlite.prepare("SELECT COUNT(*) AS n FROM benchmark_profiles WHERE model LIKE 'race/%'").get() + ).toEqual({ n: 1 }); + }); }); diff --git a/services/auto-routing-benchmark/src/profiles.test.ts b/services/auto-routing-benchmark/src/profiles.test.ts index ae21142742c..1827ca975cf 100644 --- a/services/auto-routing-benchmark/src/profiles.test.ts +++ b/services/auto-routing-benchmark/src/profiles.test.ts @@ -22,6 +22,7 @@ type EventRow = { type PendingStmt = | { kind: 'event'; values: Omit; guarded: boolean } | { kind: 'claim_user_requested'; entries: string[] } + | { kind: 'quota_guard' } | { kind: 'profile'; values: ProfileRow; @@ -273,6 +274,11 @@ vi.mock('drizzle-orm/d1', () => { // against real SQLite in profiles-sql.test.ts. return; } + if (stmt.kind === 'quota_guard') { + // Not simulated: the in-batch quota check runs against real SQLite in + // profile-quota-sql.test.ts. + return; + } if (stmt.kind === 'event') { if (stmt.guarded && hasActiveCurrentProfile(stmt.values)) { // INSERT...SELECT...WHERE NOT EXISTS → zero rows. diff --git a/services/auto-routing-benchmark/src/profiles.ts b/services/auto-routing-benchmark/src/profiles.ts index 1886fbd1eff..5158d267673 100644 --- a/services/auto-routing-benchmark/src/profiles.ts +++ b/services/auto-routing-benchmark/src/profiles.ts @@ -14,7 +14,7 @@ import { } from '@kilocode/auto-routing-contracts'; import type { BatchItem } from 'drizzle-orm/batch'; import { and, asc, eq, gte, inArray, or, sql } from 'drizzle-orm'; -import { drizzle } from 'drizzle-orm/d1'; +import { drizzle, type DrizzleD1Database } from 'drizzle-orm/d1'; import { benchmarkProfiles, profileRequestEvents } from './db-schema'; import { variantFromStorage, variantToStorage } from './reasoning-effort'; import { computeEngineIdentity } from './run'; @@ -424,6 +424,69 @@ export function chargedEventInsertStatement( ); } +type ProfileRequestEventRow = typeof profileRequestEvents.$inferSelect; + +type AdmissionQuotaScope = { + ownerType: ProfileOwnerType; + ownerId: string; + router: AutoRouter; + windowStartIso: string; +}; + +/** + * Last statement of a charged admission batch. SQLite has no RAISE outside triggers, so + * it fails the batch with a NOT NULL violation when the batch's own events pushed the + * rolling count over the limit; D1 then rolls back every write of that batch. Writer + * serialization makes concurrent admissions count each other's committed events. + */ +function admissionQuotaGuardStatement(orm: DrizzleD1Database, scope: AdmissionQuotaScope) { + return Object.assign( + orm.insert(profileRequestEvents).select(sql` + SELECT NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + WHERE ( + SELECT COUNT(*) FROM ${profileRequestEvents} + WHERE ${profileRequestEvents.owner_type} = ${scope.ownerType} + AND ${profileRequestEvents.owner_id} = ${scope.ownerId} + AND ${profileRequestEvents.router} = ${scope.router} + AND ${profileRequestEvents.admitted_at} >= ${scope.windowStartIso} + ) > ${PROFILE_ADMISSION_LIMIT} + `), + { __stmt: { kind: 'quota_guard' as const } } + ); +} + +function recentAdmissionEvents( + orm: DrizzleD1Database, + scope: AdmissionQuotaScope +): Promise { + return orm + .select() + .from(profileRequestEvents) + .where( + and( + eq(profileRequestEvents.owner_type, scope.ownerType), + eq(profileRequestEvents.owner_id, scope.ownerId), + eq(profileRequestEvents.router, scope.router), + gte(profileRequestEvents.admitted_at, scope.windowStartIso) + ) + ) + .orderBy(asc(profileRequestEvents.admitted_at)); +} + +function profileQuotaExceeded( + recentEvents: ReadonlyArray<{ admitted_at: string }>, + now: Date +): ProfileQuotaExceededError { + const retryAt = computeQuotaRetryAt( + recentEvents[0]?.admitted_at ?? now.toISOString(), + now.getTime() + ); + return new ProfileQuotaExceededError({ + error: `Profile benchmark request limit reached. New benchmarks can be requested after ${retryAt}.`, + retryAt, + }); +} + export type RegisterProfilesInput = { ownerType: ProfileOwnerType; ownerId: string; @@ -491,26 +554,15 @@ export async function registerProfiles( } const orm = drizzle(db); - const recentEvents = await orm - .select() - .from(profileRequestEvents) - .where( - and( - eq(profileRequestEvents.owner_type, input.ownerType), - eq(profileRequestEvents.owner_id, input.ownerId), - eq(profileRequestEvents.router, router), - gte(profileRequestEvents.admitted_at, windowStartIso) - ) - ) - .orderBy(asc(profileRequestEvents.admitted_at)); - + const quotaScope: AdmissionQuotaScope = { + ownerType: input.ownerType, + ownerId: input.ownerId, + router, + windowStartIso, + }; + const recentEvents = await recentAdmissionEvents(orm, quotaScope); if (recentEvents.length + chargedModels.size > PROFILE_ADMISSION_LIMIT) { - const oldest = recentEvents[0]?.admitted_at ?? nowIso; - const retryAt = computeQuotaRetryAt(oldest, now.getTime()); - throw new ProfileQuotaExceededError({ - error: `Profile benchmark request limit reached. New benchmarks can be requested after ${retryAt}.`, - retryAt, - }); + throw profileQuotaExceeded(recentEvents, now); } const admissions = [...chargedAdmissions, ...freeAdmissions]; @@ -552,7 +604,20 @@ export async function registerProfiles( claimUserRequestedStatement(orm, input.pairs, current, nowIso) as BatchItem<'sqlite'> ); - await orm.batch(stmts as [BatchItem<'sqlite'>, ...BatchItem<'sqlite'>[]]); + if (chargedModels.size > 0) { + stmts.push(admissionQuotaGuardStatement(orm, quotaScope) as BatchItem<'sqlite'>); + } + + try { + await orm.batch(stmts as [BatchItem<'sqlite'>, ...BatchItem<'sqlite'>[]]); + } catch (error) { + if (chargedModels.size === 0) throw error; + const committedEvents = await recentAdmissionEvents(orm, quotaScope); + if (committedEvents.length + chargedModels.size > PROFILE_ADMISSION_LIMIT) { + throw profileQuotaExceeded(committedEvents, now); + } + throw error; + } } // Derive reported statuses for admitted entries from post-batch rows so a diff --git a/services/auto-routing/src/custom-router-decision.ts b/services/auto-routing/src/custom-router-decision.ts index d451dd012ce..7bd59da02f5 100644 --- a/services/auto-routing/src/custom-router-decision.ts +++ b/services/auto-routing/src/custom-router-decision.ts @@ -21,6 +21,7 @@ import { CustomRouterError } from './caller-auth'; import type { CustomRouterOwner } from './custom-router-store'; import { customRouterDecisionScope } from './decision-cache'; import type { DecisionRun, DecisionTarget } from './decide'; +import { contextProvablyTooSmall, satisfiesRequiredModalities } from './decision-engine'; import { getModelCapabilities, type ModelCapabilities } from './model-capabilities'; const CUSTOM_ROUTER_TABLE_CACHE_TTL_MS = 60_000; @@ -175,7 +176,7 @@ function fallbackVariant(entry: CustomRouterEntry): string | null { } // A custom router never routes outside its pool: a null decision takes the default entry, then -// the first entry this client can serve (ADR 0005). +// the first entry this client can serve that fits the request (ADR 0005). export function customRouterDecision( routing: CustomRouting, run: DecisionRun @@ -191,7 +192,7 @@ export function customRouterDecision( const fallback = [ ...entries.filter(entry => customRouterEntryKey(entry) === defaultEntryKey), ...entries, - ].find(entry => routing.isServable(entry)); + ].find(entry => routing.isServable(entry) && fitsRequest(entry, run)); if (!fallback) { throw new CustomRouterError( 'custom_router_unavailable', @@ -215,6 +216,18 @@ export function customRouterDecision( }; } +// Same capability rules as the decision engine, without its largest-context rescue: a +// fallback entry that is inactive, lacks a required modality, or is provably too small fails. +function fitsRequest(entry: CustomRouterEntry, run: DecisionRun): boolean { + const caps = run.capabilities.get(customRouterCandidateModel(entry)); + if (caps?.isActive === false) return false; + if (run.constraints === undefined) return true; + return ( + satisfiesRequiredModalities(caps, run.constraints) && + !contextProvablyTooSmall(caps, run.constraints) + ); +} + export function publicCustomRouterDecision(decision: AutoRoutingDecision): CustomRouterDecision { const entry = decision.entry ?? parseCustomRouterCandidateModel(decision.model); if (!entry) throw new Error(`custom router decision ${decision.model} names no pool entry`); diff --git a/services/auto-routing/src/custom-router.test.ts b/services/auto-routing/src/custom-router.test.ts index 5538b4d5b51..2e5af349645 100644 --- a/services/auto-routing/src/custom-router.test.ts +++ b/services/auto-routing/src/custom-router.test.ts @@ -522,6 +522,60 @@ describe('public custom router decide', () => { }); }); + const imageMessages = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Fix the layout in this screenshot.' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } }, + ], + }, + ]; + + it('falls back past a default entry that lacks a required modality', async () => { + findCustomRouter.mockResolvedValue( + customRouter({ entries: [ollama, sonnet], defaultEntryKey: 'client:ollama/qwen3:32b' }) + ); + classifyNormalizedInput.mockRejectedValue(new Error('classifier down')); + const { decision } = await decisionOf( + await post(CUSTOM_ROUTER_DECIDE_PATH, decideBody({ messages: imageMessages })) + ); + expect(decision).toMatchObject({ model: SONNET, fallback: true }); + }); + + it('fails with custom_router_unavailable when the classifier fails and no entry fits', async () => { + findCustomRouter.mockResolvedValue( + customRouter({ entries: [ollama, lmstudio], defaultEntryKey: 'client:ollama/qwen3:32b' }) + ); + classifyNormalizedInput.mockRejectedValue(new Error('classifier down')); + const res = await post( + CUSTOM_ROUTER_DECIDE_PATH, + decideBody({ + messages: imageMessages, + clientEntries: [ + { provider: 'ollama', model: 'qwen3:32b' }, + { provider: 'lmstudio', model: 'gemma' }, + ], + }) + ); + expect(res.status).toBe(422); + expect(await errorCode(res)).toBe('custom_router_unavailable'); + }); + + it('accepts a body at the size limit and rejects one byte more', async () => { + const limit = 32_000_000; + const padded = (bytes: number) => { + const unpadded = JSON.stringify({ ...decideBody(), padding: '' }).length; + return { ...decideBody(), padding: 'x'.repeat(bytes - unpadded) }; + }; + await decisionOf(await post(CUSTOM_ROUTER_DECIDE_PATH, padded(limit))); + const tooLarge = await post(CUSTOM_ROUTER_DECIDE_PATH, padded(limit + 1)); + expect(tooLarge.status).toBe(400); + expect(await tooLarge.json()).toEqual({ + error: { code: 'invalid_request', message: 'Request body is too large' }, + }); + }); + it('drops gateway entries the organization policy denies the member', async () => { gatewayDeniedModelIds = [SONNET]; const { decision } = await decisionOf( diff --git a/services/auto-routing/src/decide.ts b/services/auto-routing/src/decide.ts index ab25ba3978d..512bcc6f49a 100644 --- a/services/auto-routing/src/decide.ts +++ b/services/auto-routing/src/decide.ts @@ -419,6 +419,8 @@ export type DecisionTarget = { export type DecisionRun = { response: AutoRoutingDecisionResponse; routingTable: RoutingTable | null; + capabilities: ModelCapabilitiesMap; + constraints: RoutingConstraints | undefined; }; function platformTarget( @@ -756,6 +758,8 @@ export async function runDecision( return { response: { cost: 0, decision: outcome.decision, classifierResult: null }, routingTable, + capabilities, + constraints, }; } if (outcome.kind === 'cache_hit') { @@ -823,7 +827,12 @@ export async function runDecision( shadow ); } - return { response: decisionResponse(cached, payload.input, decision), routingTable }; + return { + response: decisionResponse(cached, payload.input, decision), + routingTable, + capabilities, + constraints, + }; } try { @@ -886,6 +895,8 @@ export async function runDecision( return { response: decisionResponse(classifier.classification, payload.input, decision), routingTable, + capabilities, + constraints, }; } catch (error) { recordDecision( @@ -898,7 +909,7 @@ export async function runDecision( sticky, speculative ? 'speculative' : 'sync' ); - return { response: emptyDecisionResponse(), routingTable }; + return { response: emptyDecisionResponse(), routingTable, capabilities, constraints }; } } diff --git a/services/auto-routing/src/public-routes.ts b/services/auto-routing/src/public-routes.ts index 5545984bdc6..6a3bbf16b9f 100644 --- a/services/auto-routing/src/public-routes.ts +++ b/services/auto-routing/src/public-routes.ts @@ -22,6 +22,7 @@ import { import type { OrganizationRole } from '@kilocode/db'; import { formatError, getCachedSecret } from '@kilocode/worker-utils'; import { canSpendPaidUsage } from '@kilocode/worker-utils/account-access'; +import { readBoundedStream } from '@kilocode/worker-utils/bounded-stream-reader'; import type { Context, Handler } from 'hono'; import type * as z from 'zod'; import { customRouterBenchmarkContext, forwardClientEval } from './benchmark-origin'; @@ -64,10 +65,21 @@ function publicHandler(handle: (c: Context) => Promise): Hand }; } -async function readJson(c: Context): Promise<{ text: string; json: unknown }> { - const text = await c.req.text(); +// A decide request carries the whole conversation, inline images included. The limit matches +// the largest request an upstream model provider accepts (Anthropic Messages, 32 MB). +const MAX_PUBLIC_REQUEST_BODY_BYTES = 32 * 1000 * 1000; + +async function readJson(c: Context): Promise<{ byteLength: number; json: unknown }> { + const body = c.req.raw.body; + const read = body + ? await readBoundedStream(body as ReadableStream, MAX_PUBLIC_REQUEST_BODY_BYTES) + : { ok: true as const, bytes: new Uint8Array() }; + if (!read.ok) throw new CustomRouterError('invalid_request', 'Request body is too large'); try { - return { text, json: JSON.parse(text) }; + return { + byteLength: read.bytes.byteLength, + json: JSON.parse(new TextDecoder().decode(read.bytes)), + }; } catch { throw new CustomRouterError('invalid_request', 'Invalid JSON body'); } @@ -92,7 +104,7 @@ function requirePaidUsageBalance(balanceUsd: number): void { export const publicDecideHandler = publicHandler(async c => { const caller = await authenticateCaller(c.env, c.req.raw.headers); - const { text, json } = await readJson(c); + const { byteLength, json } = await readJson(c); const request = parseBody(CustomRouterDecideRequestSchema, json); const reference = parseCustomRouterModelId(request.model); if (!reference) throw new CustomRouterError('invalid_request', 'Not a custom router model ID'); @@ -160,7 +172,7 @@ export const publicDecideHandler = publicHandler(async c => { clientRequestId: request.clientRequestId ?? null, mode: request.mode ?? null, userAgent: c.req.header('user-agent') ?? null, - bodyBytes: new TextEncoder().encode(text).byteLength, + bodyBytes: byteLength, customRouter: reference, clientEntries: request.clientEntries, };