From 2e12af1db998223f0e0c54a099b03442901cc3ae Mon Sep 17 00:00:00 2001 From: AndriySvyryd <6539701+AndriySvyryd@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:02:38 +0000 Subject: [PATCH 1/2] Fix Vally experiment judge timeout forwarding - Forward the native experiment timeout flag only with integrated comparison\n- Preserve grading adapter timeouts and cover native report and command contracts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/harness-evaluation/src/vally.mjs | 6 ++++-- eng/harness-evaluation/test/harness.test.mjs | 22 ++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/eng/harness-evaluation/src/vally.mjs b/eng/harness-evaluation/src/vally.mjs index 7984f68dc5a..e94be0bc5b5 100644 --- a/eng/harness-evaluation/src/vally.mjs +++ b/eng/harness-evaluation/src/vally.mjs @@ -10,8 +10,10 @@ if (process.argv[2] === '--judge-timeout') { const judgeTimeout = process.argv[3]; timeoutMs = timeoutMilliseconds(judgeTimeout, '--judge-timeout'); process.argv.splice(2, 2); - const commandIndex = process.argv[2] === 'experiment' && process.argv[3] === 'run' ? 4 : 3; - process.argv.splice(commandIndex, 0, '--judge-timeout', judgeTimeout); + const experimentRun = process.argv[2] === 'experiment' && process.argv[3] === 'run'; + if (!experimentRun || process.argv.includes('--compare')) { + process.argv.splice(experimentRun ? 4 : 3, 0, '--judge-timeout', judgeTimeout); + } } judgeTimeoutLimits(timeoutMs); diff --git a/eng/harness-evaluation/test/harness.test.mjs b/eng/harness-evaluation/test/harness.test.mjs index 9f6574ca526..f4cee78699c 100644 --- a/eng/harness-evaluation/test/harness.test.mjs +++ b/eng/harness-evaluation/test/harness.test.mjs @@ -83,7 +83,7 @@ variants: const outputRoot = join(root, 'artifacts/results'); const execution = runVally(['experiment', 'run', experimentPath, '--output-dir', outputRoot, '--param', 'RUNS=1', '--workers', '1'], { cwd: root, judgeTimeout: '10m' }); - assert.ok([0, 1].includes(execution.status), execution.stderr); + assert.equal(execution.status, 0, execution.stderr + execution.stdout); const experimentDirectory = await findExperimentRunDirectory(outputRoot); const planPath = join(experimentDirectory, 'plan-snapshot.json'); const plan = JSON.parse(await readFile(planPath, 'utf8')); @@ -93,6 +93,8 @@ variants: for (const variant of ['control', 'treatment']) { const resultsPath = join(experimentDirectory, variant, 'results.jsonl'); assert.equal(await variantCompleted(resultsPath, planPath, variant), true, await readFile(resultsPath, 'utf8')); + assert.equal(await variantPassed(resultsPath, + join(root, 'eng/harness-evaluation/skills/example/eval.yaml'), planPath, variant), true); const records = (await readFile(resultsPath, 'utf8')).split(/\r?\n/).filter(Boolean).map(JSON.parse); const trials = records.filter((record) => record.type === 'trial-result'); assert.equal(trials.length, 1); @@ -535,8 +537,12 @@ test('judge timeout loader preserves explicit timeouts without changing the nati }`); await writeFile(preloadPath, `import { register } from 'node:module'; register(${JSON.stringify(pathToFileURL(loaderPath).href)}, import.meta.url);`); - for (const [command, subcommand, fixture] of [['experiment', 'run', 'fixture'], ['compare', undefined, 'fixture']]) { - const commandArguments = subcommand ? [command, subcommand, fixture] : [command, fixture]; + for (const commandArguments of [ + ['experiment', 'run', 'fixture'], + ['experiment', 'run', 'fixture', '--compare'], + ['compare', 'fixture'], + ['eval', 'fixture'], + ]) { for (const [timeout, expected] of [[undefined, 600_000], ['15m', 900_000], ['30s', 30_000]]) { const result = spawnSync(process.execPath, ['--import', pathToFileURL(preloadPath).href, fileURLToPath(new URL('../src/vally.mjs', import.meta.url)), @@ -546,9 +552,13 @@ test('judge timeout loader preserves explicit timeouts without changing the nati const lines = result.stdout.trim().split(/\r?\n/).map(JSON.parse); assert.deepEqual(lines, [ { sdkTimeoutMs: expected, verdict: { passed: true } }, - subcommand - ? [command, subcommand, ...(timeout ? ['--judge-timeout', timeout] : []), fixture] - : [command, ...(timeout ? ['--judge-timeout', timeout] : []), fixture], + timeout && (commandArguments[0] !== 'experiment' || commandArguments.includes('--compare')) + ? [ + ...commandArguments.slice(0, commandArguments[0] === 'experiment' ? 2 : 1), + '--judge-timeout', timeout, + ...commandArguments.slice(commandArguments[0] === 'experiment' ? 2 : 1), + ] + : commandArguments, ]); } } From faae8ec5bbf5f7cabe3ed64bec5dddc303d0df80 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Fri, 30 Oct 2026 17:59:28 -0700 Subject: [PATCH 2/2] Update eval models and lower judge reasoning effort Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/harness-evaluation/README.md | 2 ++ .../instructions/copilot-instructions/eval.yaml | 6 +++--- .../skills/change-tracking/eval.yaml | 6 +++--- .../skills/code-review/eval.yaml | 6 +++--- .../skills/cosmos-provider/eval.yaml | 6 +++--- .../skills/make-custom-agent/eval.yaml | 6 +++--- .../make-github-actions-workflow/eval.yaml | 6 +++--- .../skills/make-instructions/eval.yaml | 6 +++--- .../skills/make-skill/eval.yaml | 6 +++--- .../skills/migrations/eval.yaml | 6 +++--- .../skills/model-building/eval.yaml | 6 +++--- .../skills/query-pipeline/eval.yaml | 6 +++--- .../skills/run-apichief/eval.yaml | 6 +++--- .../skills/scaffolding/eval.yaml | 6 +++--- .../skills/servicing-pr/eval.yaml | 6 +++--- .../skills/sqlite-adonet/eval.yaml | 6 +++--- eng/harness-evaluation/skills/tooling/eval.yaml | 6 +++--- eng/harness-evaluation/skills/triage/eval.yaml | 6 +++--- .../skills/update-pipeline/eval.yaml | 6 +++--- eng/harness-evaluation/test/harness.test.mjs | 17 ++++++++++------- 20 files changed, 66 insertions(+), 61 deletions(-) diff --git a/eng/harness-evaluation/README.md b/eng/harness-evaluation/README.md index f2f50c63ec1..f0ec773f224 100644 --- a/eng/harness-evaluation/README.md +++ b/eng/harness-evaluation/README.md @@ -101,6 +101,8 @@ The evals need no running emulator or database. Do not launch infrastructure or Behavioral runs use the Copilot SDK and require a valid local Copilot login or `COPILOT_GITHUB_TOKEN`. Results are written below `artifacts/TestResults/harness-evaluation//` as one timestamped experiment containing separate `control` and `treatment` variants, plus `comparison.jsonl`. They may contain prompts, model output, and tool payloads; do not commit them. +All evals default to `gpt-6.1-terra` for agent execution and `gpt-6.1-sol` with `low` reasoning effort for grading and paired comparison. Use `--model`, `--judge-model`, and `--judge-reasoning-effort` to override these defaults locally. + ### Agent execution `cli.mjs eval` produces large progress and grading reports. Agents must use their execution environment's background mode and redirect stdout and stderr to files; do not stream command output into the conversation context. A PowerShell fallback from `eng/harness-evaluation` is: diff --git a/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml b/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml index af32d576897..b13127c2c96 100644 --- a/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml +++ b/eng/harness-evaluation/instructions/copilot-instructions/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates repository-wide Copilot guidance while implementing EF Co type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/change-tracking/eval.yaml b/eng/harness-evaluation/skills/change-tracking/eval.yaml index 34f7f724502..6d572426a9a 100644 --- a/eng/harness-evaluation/skills/change-tracking/eval.yaml +++ b/eng/harness-evaluation/skills/change-tracking/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates change-tracking guidance against EF Core snapshot and ent type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/code-review/eval.yaml b/eng/harness-evaluation/skills/code-review/eval.yaml index be03a1558e5..579203894cb 100644 --- a/eng/harness-evaluation/skills/code-review/eval.yaml +++ b/eng/harness-evaluation/skills/code-review/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates evidence-backed EF Core review behavior, API compatibilit type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/cosmos-provider/eval.yaml b/eng/harness-evaluation/skills/cosmos-provider/eval.yaml index 5ae88b41922..5f76783edd5 100644 --- a/eng/harness-evaluation/skills/cosmos-provider/eval.yaml +++ b/eng/harness-evaluation/skills/cosmos-provider/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates Cosmos provider guidance when adding transactional batch type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/make-custom-agent/eval.yaml b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml index 92ab5b9882d..1013f26562c 100644 --- a/eng/harness-evaluation/skills/make-custom-agent/eval.yaml +++ b/eng/harness-evaluation/skills/make-custom-agent/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates custom-agent review boundaries and isolated behavioral co type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml b/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml index a7d54a1103f..3fb4a99e744 100644 --- a/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml +++ b/eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates workflow authoring against EF Core's fork-safe PR automat type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/make-instructions/eval.yaml b/eng/harness-evaluation/skills/make-instructions/eval.yaml index 5084aad300d..ca81e8e5ffc 100644 --- a/eng/harness-evaluation/skills/make-instructions/eval.yaml +++ b/eng/harness-evaluation/skills/make-instructions/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates file-instruction authoring against EF Core query implemen type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/make-skill/eval.yaml b/eng/harness-evaluation/skills/make-skill/eval.yaml index 1928b7ab4fe..dcc5582a73d 100644 --- a/eng/harness-evaluation/skills/make-skill/eval.yaml +++ b/eng/harness-evaluation/skills/make-skill/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates correction of an inaccurate EF Core skill and its paired type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/migrations/eval.yaml b/eng/harness-evaluation/skills/migrations/eval.yaml index 094bd3072e9..b21d0e9888a 100644 --- a/eng/harness-evaluation/skills/migrations/eval.yaml +++ b/eng/harness-evaluation/skills/migrations/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates migrations guidance against EF Core model differ behavior type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/model-building/eval.yaml b/eng/harness-evaluation/skills/model-building/eval.yaml index f66754513c4..2e97a9186b8 100644 --- a/eng/harness-evaluation/skills/model-building/eval.yaml +++ b/eng/harness-evaluation/skills/model-building/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates model-building guidance against EF Core convention ambigu type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/query-pipeline/eval.yaml b/eng/harness-evaluation/skills/query-pipeline/eval.yaml index 8c36ec30e19..0806ed9ecc0 100644 --- a/eng/harness-evaluation/skills/query-pipeline/eval.yaml +++ b/eng/harness-evaluation/skills/query-pipeline/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates optimization safety, translation fallback isolation, spli type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/run-apichief/eval.yaml b/eng/harness-evaluation/skills/run-apichief/eval.yaml index fc7519a981d..b3c847e842f 100644 --- a/eng/harness-evaluation/skills/run-apichief/eval.yaml +++ b/eng/harness-evaluation/skills/run-apichief/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates ApiChief workflow guidance against the EFCore public API type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/scaffolding/eval.yaml b/eng/harness-evaluation/skills/scaffolding/eval.yaml index dbc99e71ef8..6269dad982d 100644 --- a/eng/harness-evaluation/skills/scaffolding/eval.yaml +++ b/eng/harness-evaluation/skills/scaffolding/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates reverse-engineering guidance against EF Core scaffolding type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/servicing-pr/eval.yaml b/eng/harness-evaluation/skills/servicing-pr/eval.yaml index 69261e31536..89daefec82e 100644 --- a/eng/harness-evaluation/skills/servicing-pr/eval.yaml +++ b/eng/harness-evaluation/skills/servicing-pr/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates servicing PR guidance using EF Core branch policy, source type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml b/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml index 80142a11e14..2d0d97a27b8 100644 --- a/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml +++ b/eng/harness-evaluation/skills/sqlite-adonet/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates Microsoft.Data.Sqlite guidance against its command and co type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/tooling/eval.yaml b/eng/harness-evaluation/skills/tooling/eval.yaml index e0b6f524873..a09ce10824f 100644 --- a/eng/harness-evaluation/skills/tooling/eval.yaml +++ b/eng/harness-evaluation/skills/tooling/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates dotnet-ef tooling guidance against the EF Core command im type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/triage/eval.yaml b/eng/harness-evaluation/skills/triage/eval.yaml index e90b32ac03c..a034fb3a66b 100644 --- a/eng/harness-evaluation/skills/triage/eval.yaml +++ b/eng/harness-evaluation/skills/triage/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates evidence-backed EF Core triage, untrusted-report safety, type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/skills/update-pipeline/eval.yaml b/eng/harness-evaluation/skills/update-pipeline/eval.yaml index b5e284dc9cd..41dd3409084 100644 --- a/eng/harness-evaluation/skills/update-pipeline/eval.yaml +++ b/eng/harness-evaluation/skills/update-pipeline/eval.yaml @@ -3,9 +3,9 @@ description: Evaluates SaveChanges update-pipeline guidance against EF Core batc type: capability defaults: runs: ${RUNS=5} - model: ${MODEL=gpt-6.1-sol} - judge_model: ${JUDGE_MODEL=gpt-5.6-terra} - judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium} + model: ${MODEL=gpt-6.1-terra} + judge_model: ${JUDGE_MODEL=gpt-6.1-sol} + judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low} judge_timeout: ${JUDGE_TIMEOUT=10m} executor: copilot-sdk agent_environment: diff --git a/eng/harness-evaluation/test/harness.test.mjs b/eng/harness-evaluation/test/harness.test.mjs index f4cee78699c..b3054868490 100644 --- a/eng/harness-evaluation/test/harness.test.mjs +++ b/eng/harness-evaluation/test/harness.test.mjs @@ -615,19 +615,21 @@ test('judge timeout loader scales comparison deadlines and fails closed on nativ } }); -test('all evals specify and resolve judge defaults and overrides through Vally', async () => { +test('all evals specify and resolve model and judge defaults and overrides through Vally', async () => { const components = await discoverComponents(); assert.ok(components.length > 0); for (const component of components) { const evalPath = join(defaultRepoRoot, component.eval); const authored = parse(await readFile(evalPath, 'utf8')); - assert.equal(authored.defaults.judge_model, '${JUDGE_MODEL=gpt-5.6-terra}', component.id); - assert.equal(authored.defaults.judge_reasoning_effort, '${JUDGE_REASONING_EFFORT=medium}', component.id); + assert.equal(authored.defaults.model, '${MODEL=gpt-6.1-terra}', component.id); + assert.equal(authored.defaults.judge_model, '${JUDGE_MODEL=gpt-6.1-sol}', component.id); + assert.equal(authored.defaults.judge_reasoning_effort, '${JUDGE_REASONING_EFFORT=low}', component.id); assert.equal(authored.defaults.judge_timeout, '${JUDGE_TIMEOUT=10m}', component.id); assert.equal(authored.scoring.timeout, '${SCORING_TIMEOUT=10m}', component.id); const defaults = (await loadEvalWithParams(evalPath)).spec.defaults; - assert.equal(defaults.judge_reasoning_effort, 'medium', component.id); - assert.equal(defaults.judge_model, 'gpt-5.6-terra', component.id); + assert.equal(defaults.judge_reasoning_effort, 'low', component.id); + assert.equal(defaults.model, 'gpt-6.1-terra', component.id); + assert.equal(defaults.judge_model, 'gpt-6.1-sol', component.id); assert.equal(defaults.judge_timeout, '10m', component.id); assert.equal(authored.defaults.timeout, undefined, component.id); assert.equal(defaults.timeout, undefined, component.id); @@ -639,10 +641,11 @@ test('all evals specify and resolve judge defaults and overrides through Vally', `${component.id}/${stimulus.name}`); } const overridden = (await loadEvalWithParams(evalPath, { - cliParams: { JUDGE_REASONING_EFFORT: 'high', JUDGE_MODEL: 'gpt-6.1-sol', JUDGE_TIMEOUT: '12m', SCORING_TIMEOUT: '15m', EVAL_TIMEOUT: '45m' }, + cliParams: { MODEL: 'gpt-6.1-sol', JUDGE_REASONING_EFFORT: 'high', JUDGE_MODEL: 'gpt-6.1-terra', JUDGE_TIMEOUT: '12m', SCORING_TIMEOUT: '15m', EVAL_TIMEOUT: '45m' }, })).spec; assert.equal(overridden.defaults.judge_reasoning_effort, 'high', component.id); - assert.equal(overridden.defaults.judge_model, 'gpt-6.1-sol'); + assert.equal(overridden.defaults.model, 'gpt-6.1-sol', component.id); + assert.equal(overridden.defaults.judge_model, 'gpt-6.1-terra', component.id); assert.equal(overridden.defaults.judge_timeout, '12m', component.id); assert.equal(overridden.scoring.timeout, '15m', component.id); assert.equal(overridden.defaults.timeout, undefined, component.id);