Fix Vally experiment judge timeout forwarding

eng/harness-evaluation/README.md

@@ -101,6 +101,8 @@ The evals need no running emulator or database. Do not launch infrastructure or
 
 Behavioral runs use the Copilot SDK and require a valid local Copilot login or `COPILOT_GITHUB_TOKEN`. Results are written below `artifacts/TestResults/harness-evaluation/<component-id>/` as one timestamped experiment containing separate `control` and `treatment` variants, plus `comparison.jsonl`. They may contain prompts, model output, and tool payloads; do not commit them.
 
+All evals default to `gpt-6.1-terra` for agent execution and `gpt-6.1-sol` with `low` reasoning effort for grading and paired comparison. Use `--model`, `--judge-model`, and `--judge-reasoning-effort` to override these defaults locally.
+
 ### Agent execution
 
 `cli.mjs eval` produces large progress and grading reports. Agents must use their execution environment's background mode and redirect stdout and stderr to files; do not stream command output into the conversation context. A PowerShell fallback from `eng/harness-evaluation` is:

eng/harness-evaluation/instructions/copilot-instructions/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates repository-wide Copilot guidance while implementing EF Co
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/change-tracking/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates change-tracking guidance against EF Core snapshot and ent
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/code-review/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates evidence-backed EF Core review behavior, API compatibilit
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/cosmos-provider/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates Cosmos provider guidance when adding transactional batch
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/make-custom-agent/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates custom-agent review boundaries and isolated behavioral co
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/make-github-actions-workflow/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates workflow authoring against EF Core's fork-safe PR automat
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/make-instructions/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates file-instruction authoring against EF Core query implemen
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/make-skill/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates correction of an inaccurate EF Core skill and its paired
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/migrations/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates migrations guidance against EF Core model differ behavior
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/model-building/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates model-building guidance against EF Core convention ambigu
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/query-pipeline/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates optimization safety, translation fallback isolation, spli
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/run-apichief/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates ApiChief workflow guidance against the EFCore public API
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/scaffolding/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates reverse-engineering guidance against EF Core scaffolding
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/servicing-pr/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates servicing PR guidance using EF Core branch policy, source
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/sqlite-adonet/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates Microsoft.Data.Sqlite guidance against its command and co
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/tooling/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates dotnet-ef tooling guidance against the EF Core command im
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/triage/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates evidence-backed EF Core triage, untrusted-report safety,
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/skills/update-pipeline/eval.yaml

@@ -3,9 +3,9 @@ description: Evaluates SaveChanges update-pipeline guidance against EF Core batc
 type: capability
 defaults:
   runs: ${RUNS=5}
-  model: ${MODEL=gpt-6.1-sol}
-  judge_model: ${JUDGE_MODEL=gpt-5.6-terra}
-  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=medium}
+  model: ${MODEL=gpt-6.1-terra}
+  judge_model: ${JUDGE_MODEL=gpt-6.1-sol}
+  judge_reasoning_effort: ${JUDGE_REASONING_EFFORT=low}
   judge_timeout: ${JUDGE_TIMEOUT=10m}
   executor: copilot-sdk
 agent_environment:

eng/harness-evaluation/src/vally.mjs

@@ -10,8 +10,10 @@ if (process.argv[2] === '--judge-timeout') {
   const judgeTimeout = process.argv[3];
   timeoutMs = timeoutMilliseconds(judgeTimeout, '--judge-timeout');
   process.argv.splice(2, 2);
-  const commandIndex = process.argv[2] === 'experiment' && process.argv[3] === 'run' ? 4 : 3;
-  process.argv.splice(commandIndex, 0, '--judge-timeout', judgeTimeout);
+  const experimentRun = process.argv[2] === 'experiment' && process.argv[3] === 'run';
+  if (!experimentRun || process.argv.includes('--compare')) {
+    process.argv.splice(experimentRun ? 4 : 3, 0, '--judge-timeout', judgeTimeout);
+  }
 }
 judgeTimeoutLimits(timeoutMs);
 

eng/harness-evaluation/test/harness.test.mjs

@@ -83,7 +83,7 @@ variants:
     const outputRoot = join(root, 'artifacts/results');
     const execution = runVally(['experiment', 'run', experimentPath, '--output-dir', outputRoot,
       '--param', 'RUNS=1', '--workers', '1'], { cwd: root, judgeTimeout: '10m' });
-    assert.ok([0, 1].includes(execution.status), execution.stderr);
+    assert.equal(execution.status, 0, execution.stderr + execution.stdout);
     const experimentDirectory = await findExperimentRunDirectory(outputRoot);
     const planPath = join(experimentDirectory, 'plan-snapshot.json');
     const plan = JSON.parse(await readFile(planPath, 'utf8'));
@@ -93,6 +93,8 @@ variants:
     for (const variant of ['control', 'treatment']) {
       const resultsPath = join(experimentDirectory, variant, 'results.jsonl');
       assert.equal(await variantCompleted(resultsPath, planPath, variant), true, await readFile(resultsPath, 'utf8'));
+      assert.equal(await variantPassed(resultsPath,
+        join(root, 'eng/harness-evaluation/skills/example/eval.yaml'), planPath, variant), true);
       const records = (await readFile(resultsPath, 'utf8')).split(/\r?\n/).filter(Boolean).map(JSON.parse);
       const trials = records.filter((record) => record.type === 'trial-result');
       assert.equal(trials.length, 1);
@@ -535,8 +537,12 @@ test('judge timeout loader preserves explicit timeouts without changing the nati
     }`);
     await writeFile(preloadPath, `import { register } from 'node:module';
       register(${JSON.stringify(pathToFileURL(loaderPath).href)}, import.meta.url);`);
-    for (const [command, subcommand, fixture] of [['experiment', 'run', 'fixture'], ['compare', undefined, 'fixture']]) {
-      const commandArguments = subcommand ? [command, subcommand, fixture] : [command, fixture];
+    for (const commandArguments of [
+      ['experiment', 'run', 'fixture'],
+      ['experiment', 'run', 'fixture', '--compare'],
+      ['compare', 'fixture'],
+      ['eval', 'fixture'],
+    ]) {
       for (const [timeout, expected] of [[undefined, 600_000], ['15m', 900_000], ['30s', 30_000]]) {
         const result = spawnSync(process.execPath, ['--import', pathToFileURL(preloadPath).href,
           fileURLToPath(new URL('../src/vally.mjs', import.meta.url)),
@@ -546,9 +552,13 @@ test('judge timeout loader preserves explicit timeouts without changing the nati
         const lines = result.stdout.trim().split(/\r?\n/).map(JSON.parse);
         assert.deepEqual(lines, [
           { sdkTimeoutMs: expected, verdict: { passed: true } },
-          subcommand
-            ? [command, subcommand, ...(timeout ? ['--judge-timeout', timeout] : []), fixture]
-            : [command, ...(timeout ? ['--judge-timeout', timeout] : []), fixture],
+          timeout && (commandArguments[0] !== 'experiment' || commandArguments.includes('--compare'))
+            ? [
+              ...commandArguments.slice(0, commandArguments[0] === 'experiment' ? 2 : 1),
+              '--judge-timeout', timeout,
+              ...commandArguments.slice(commandArguments[0] === 'experiment' ? 2 : 1),
+            ]
+            : commandArguments,
         ]);
       }
     }
@@ -605,19 +615,21 @@ test('judge timeout loader scales comparison deadlines and fails closed on nativ
   }
 });
 
-test('all evals specify and resolve judge defaults and overrides through Vally', async () => {
+test('all evals specify and resolve model and judge defaults and overrides through Vally', async () => {
   const components = await discoverComponents();
   assert.ok(components.length > 0);
   for (const component of components) {
     const evalPath = join(defaultRepoRoot, component.eval);
     const authored = parse(await readFile(evalPath, 'utf8'));
-    assert.equal(authored.defaults.judge_model, '${JUDGE_MODEL=gpt-5.6-terra}', component.id);
-    assert.equal(authored.defaults.judge_reasoning_effort, '${JUDGE_REASONING_EFFORT=medium}', component.id);
+    assert.equal(authored.defaults.model, '${MODEL=gpt-6.1-terra}', component.id);
+    assert.equal(authored.defaults.judge_model, '${JUDGE_MODEL=gpt-6.1-sol}', component.id);
+    assert.equal(authored.defaults.judge_reasoning_effort, '${JUDGE_REASONING_EFFORT=low}', component.id);
     assert.equal(authored.defaults.judge_timeout, '${JUDGE_TIMEOUT=10m}', component.id);
     assert.equal(authored.scoring.timeout, '${SCORING_TIMEOUT=10m}', component.id);
     const defaults = (await loadEvalWithParams(evalPath)).spec.defaults;
-    assert.equal(defaults.judge_reasoning_effort, 'medium', component.id);
-    assert.equal(defaults.judge_model, 'gpt-5.6-terra', component.id);
+    assert.equal(defaults.judge_reasoning_effort, 'low', component.id);
+    assert.equal(defaults.model, 'gpt-6.1-terra', component.id);
+    assert.equal(defaults.judge_model, 'gpt-6.1-sol', component.id);
     assert.equal(defaults.judge_timeout, '10m', component.id);
     assert.equal(authored.defaults.timeout, undefined, component.id);
     assert.equal(defaults.timeout, undefined, component.id);
@@ -629,10 +641,11 @@ test('all evals specify and resolve judge defaults and overrides through Vally',
         `${component.id}/${stimulus.name}`);
     }
     const overridden = (await loadEvalWithParams(evalPath, {
-      cliParams: { JUDGE_REASONING_EFFORT: 'high', JUDGE_MODEL: 'gpt-6.1-sol', JUDGE_TIMEOUT: '12m', SCORING_TIMEOUT: '15m', EVAL_TIMEOUT: '45m' },
+      cliParams: { MODEL: 'gpt-6.1-sol', JUDGE_REASONING_EFFORT: 'high', JUDGE_MODEL: 'gpt-6.1-terra', JUDGE_TIMEOUT: '12m', SCORING_TIMEOUT: '15m', EVAL_TIMEOUT: '45m' },
     })).spec;
     assert.equal(overridden.defaults.judge_reasoning_effort, 'high', component.id);
-    assert.equal(overridden.defaults.judge_model, 'gpt-6.1-sol');
+    assert.equal(overridden.defaults.model, 'gpt-6.1-sol', component.id);
+    assert.equal(overridden.defaults.judge_model, 'gpt-6.1-terra', component.id);
     assert.equal(overridden.defaults.judge_timeout, '12m', component.id);
     assert.equal(overridden.scoring.timeout, '15m', component.id);
     assert.equal(overridden.defaults.timeout, undefined, component.id);