diff --git a/services/gastown/src/dos/Town.do.ts b/services/gastown/src/dos/Town.do.ts index dc90b31498c..88a139b57e2 100644 --- a/services/gastown/src/dos/Town.do.ts +++ b/services/gastown/src/dos/Town.do.ts @@ -4635,7 +4635,7 @@ export class TownDO extends DurableObject { * requests that reset the container's sleepAfter timer (#1409). */ private async refreshContainerToken(): Promise { - // Skip if no active work AND no actively-running mayor — the container is + // Skip if no running agent AND no actively-running mayor — the container is // sleeping (or about to) and doesn't need a fresh token. The token will be // refreshed when work is next dispatched (ensureContainerToken is called in // startAgentInContainer at container-dispatch.ts) and on the warm-send path @@ -4645,7 +4645,7 @@ export class TownDO extends DurableObject { // /refresh-token pings that reset sleepAfter (#1409). const mayor = agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null; const mayorAlive = mayor && (mayor.status === 'working' || mayor.status === 'stalled'); - if (!this.hasActiveWork() && !mayorAlive) return; + if (!scheduling.hasRunningAgents(this.sql) && !mayorAlive) return; const TOKEN_REFRESH_INTERVAL_MS = 60 * 60_000; const now = Date.now(); @@ -4671,7 +4671,7 @@ export class TownDO extends DurableObject { */ private async stopContainerIfIdle(): Promise { await _stopContainerIfIdle({ - hasActiveWork: () => this.hasActiveWork(), + hasRunningAgents: () => scheduling.hasRunningAgents(this.sql), isDraining: () => this._draining || this.containerStartsBlocked(), getMayor: () => agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null, getTownId: () => this.townId, @@ -5094,16 +5094,13 @@ export class TownDO extends DurableObject { } } - private async ensureContainerReady( - cachedRigList?: rigs.RigRecord[], - cachedActiveWork?: boolean - ): Promise { + private async ensureContainerReady(cachedRigList?: rigs.RigRecord[]): Promise { if (this.containerStartsBlocked()) return; const rigList = cachedRigList ?? rigs.listRigs(this.sql); if (rigList.length === 0) return; - const hasWork = cachedActiveWork ?? this.hasActiveWork(); - if (!hasWork && !this._draining) { + // Other active work starts the container when it is dispatched. + if (!scheduling.hasRunningAgents(this.sql) && !this._draining) { const newestRigAge = rigList.reduce((min, r) => { const age = Date.now() - new Date(r.created_at).getTime(); return Math.min(min, age); diff --git a/services/gastown/src/dos/town/container-idle-stop.test.ts b/services/gastown/src/dos/town/container-idle-stop.test.ts index 8740c099af4..5c413c28553 100644 --- a/services/gastown/src/dos/town/container-idle-stop.test.ts +++ b/services/gastown/src/dos/town/container-idle-stop.test.ts @@ -38,7 +38,7 @@ function makeDeps(overrides: Partial = {}): TestDeps { const events: Array<{ event: string; townId: string; reason: string; error?: string }> = []; return { - hasActiveWork: overrides.hasActiveWork ?? (() => false), + hasRunningAgents: overrides.hasRunningAgents ?? (() => false), isDraining: overrides.isDraining ?? (() => false), getMayor: overrides.getMayor ?? (() => null), getTownId: overrides.getTownId ?? (() => 'town-1'), @@ -70,8 +70,8 @@ function makeDeps(overrides: Partial = {}): TestDeps { } describe('stopContainerIfIdle', () => { - it('does not stop when town has active work', async () => { - const deps = makeDeps({ hasActiveWork: () => true }); + it('does not stop while an agent is running', async () => { + const deps = makeDeps({ hasRunningAgents: () => true }); await stopContainerIfIdle(deps); expect(deps._stopFn).not.toHaveBeenCalled(); }); diff --git a/services/gastown/src/dos/town/container-idle-stop.ts b/services/gastown/src/dos/town/container-idle-stop.ts index 03a02054430..f51f5045bdb 100644 --- a/services/gastown/src/dos/town/container-idle-stop.ts +++ b/services/gastown/src/dos/town/container-idle-stop.ts @@ -14,7 +14,7 @@ export const CONTAINER_IDLE_STOP_THRESHOLD_MS = 5 * 60_000; export const CONTAINER_IDLE_STOP_THROTTLE_MS = 2 * 60_000; export type IdleStopDeps = { - hasActiveWork: () => boolean; + hasRunningAgents: () => boolean; isDraining: () => boolean; getMayor: () => Agent | null; getTownId: () => string | null; @@ -29,7 +29,7 @@ export type IdleStopDeps = { }; export async function stopContainerIfIdle(deps: IdleStopDeps): Promise { - if (deps.hasActiveWork()) return; + if (deps.hasRunningAgents()) return; if (deps.isDraining()) return; const mayor = deps.getMayor(); @@ -71,6 +71,7 @@ export async function stopContainerIfIdle(deps: IdleStopDeps): Promise { await stub.stop(); await deps.setLastIdleStopAt(now); deps.writeEventFn({ event: 'container.idle_stop', townId, reason }); + logger.info('stopContainerIfIdle: stopped idle container', { reason }); } catch (err) { logger.warn('stopContainerIfIdle: stop() failed', { error: err instanceof Error ? err.message : String(err), diff --git a/services/gastown/src/dos/town/scheduling.ts b/services/gastown/src/dos/town/scheduling.ts index c60c7482f4f..080f2ebd83e 100644 --- a/services/gastown/src/dos/town/scheduling.ts +++ b/services/gastown/src/dos/town/scheduling.ts @@ -262,23 +262,6 @@ export function dispatchUnblockedBeads(ctx: SchedulingContext, closedBeadId: str * before). */ export function hasActiveWork(sql: SqlStorage): boolean { - // Stalled agents older than 30min no longer count as active work: they - // typically represent stuck rows (container crashed hard, /status keeps - // returning running/unknown). Keeping them in the active set would pin - // the alarm at its 5s fast cadence indefinitely. The stalled->idle - // auto-transition in reconcileAgents cleans them up after 2h 30min. - const hasActiveAgents = (): boolean => - countOf([ - ...query( - sql, - /* sql */ `SELECT COUNT(*) as cnt FROM ${agent_metadata} - WHERE ${agent_metadata.status} = 'working' - OR (${agent_metadata.status} = 'stalled' - AND ${agent_metadata.last_activity_at} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-30 minutes'))`, - [] - ), - ]) > 0; - // Idle agents that already hold a hook — the reconciler should dispatch // them on the next tick. const hasHookedIdleAgents = (): boolean => @@ -333,7 +316,7 @@ export function hasActiveWork(sql: SqlStorage): boolean { ]) > 0; return ( - hasActiveAgents() || + hasRunningAgents(sql) || hasHookedIdleAgents() || hasOpenMergeRequests() || hasOpenTriageBeads() || @@ -341,6 +324,32 @@ export function hasActiveWork(sql: SqlStorage): boolean { ); } +/** + * Returns true if an agent process should be running in the container. Only + * this keeps the container awake: other active work starts the container + * when the reconciler dispatches it, and work it never dispatches must not + * hold the container forever. + */ +export function hasRunningAgents(sql: SqlStorage): boolean { + // Stalled agents older than 30min no longer count as active work: they + // typically represent stuck rows (container crashed hard, /status keeps + // returning running/unknown). Keeping them in the active set would pin + // the alarm at its 5s fast cadence indefinitely. The stalled->idle + // auto-transition in reconcileAgents cleans them up after 2h 30min. + return ( + countOf([ + ...query( + sql, + /* sql */ `SELECT COUNT(*) as cnt FROM ${agent_metadata} + WHERE ${agent_metadata.status} = 'working' + OR (${agent_metadata.status} = 'stalled' + AND ${agent_metadata.last_activity_at} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-30 minutes'))`, + [] + ), + ]) > 0 + ); +} + /** Read the `cnt` column off the first row of a `SELECT COUNT(*) as cnt` query. */ function countOf(rows: ReadonlyArray>): number { return Number(rows[0]?.cnt ?? 0); diff --git a/services/gastown/test/integration/container-idle.test.ts b/services/gastown/test/integration/container-idle.test.ts new file mode 100644 index 00000000000..833c8343a5d --- /dev/null +++ b/services/gastown/test/integration/container-idle.test.ts @@ -0,0 +1,55 @@ +import { env, runDurableObjectAlarm, runInDurableObject } from 'cloudflare:test'; +import { describe, expect, it } from 'vitest'; +import { getTownContainerStub } from '../../src/dos/TownContainer.do'; +import { HELD_LABEL } from '../../src/dos/town/patrol'; + +// Older than the 5-minute window in which a new rig keeps the container warm. +const OLD_RIG_CREATED_AT = new Date(Date.now() - 60 * 60_000).toISOString(); + +async function townWithRunningContainer() { + const townId = `container-idle-${crypto.randomUUID()}`; + const town = env.TOWN.get(env.TOWN.idFromName(townId)); + await town.setTownId(townId); + await town.addRig({ + rigId: 'rig-1', + name: 'Rig', + gitUrl: 'https://github.com/org/repo.git', + defaultBranch: 'main', + }); + await runInDurableObject(town, (_instance, state) => { + state.storage.sql.exec('UPDATE rigs SET created_at = ?', OLD_RIG_CREATED_AT); + return state.storage.setAlarm(Date.now() + 60_000); + }); + const container = getTownContainerStub(env, townId); + await container.warmUp(); + return { townId, town, container }; +} + +describe('Town alarm idle container stop', () => { + it('stops the container when the town only has work it never dispatches', async () => { + const { town, container } = await townWithRunningContainer(); + // A held issue counts as active work, but the reconciler never dispatches it. + await town.createBead({ type: 'issue', title: 'Held', rig_id: 'rig-1', labels: [HELD_LABEL] }); + + expect(await runDurableObjectAlarm(town)).toBe(true); + + expect(await container.getState()).toEqual({ status: 'stopped' }); + }); + + it('keeps the container while an agent is working', async () => { + const { townId, town, container } = await townWithRunningContainer(); + const agent = await town.registerAgent({ + role: 'polecat', + name: 'P1', + identity: `polecat-${townId}`, + }); + const bead = await town.createBead({ type: 'issue', title: 'Work', rig_id: 'rig-1' }); + await town.hookBead(agent.id, bead.bead_id); + await town.updateAgentStatus(agent.id, 'working'); + await town.touchAgentHeartbeat(agent.id); + + expect(await runDurableObjectAlarm(town)).toBe(true); + + expect(await container.getState()).toEqual({ status: 'healthy' }); + }); +});