Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions services/gastown/src/dos/Town.do.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4635,7 +4635,7 @@ export class TownDO extends DurableObject<Env> {
* requests that reset the container's sleepAfter timer (#1409).
*/
private async refreshContainerToken(): Promise<void> {
// Skip if no active work AND no actively-running mayor — the container is
// Skip if no running agent AND no actively-running mayor — the container is
// sleeping (or about to) and doesn't need a fresh token. The token will be
// refreshed when work is next dispatched (ensureContainerToken is called in
// startAgentInContainer at container-dispatch.ts) and on the warm-send path
Expand All @@ -4645,7 +4645,7 @@ export class TownDO extends DurableObject<Env> {
// /refresh-token pings that reset sleepAfter (#1409).
const mayor = agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null;
const mayorAlive = mayor && (mayor.status === 'working' || mayor.status === 'stalled');
if (!this.hasActiveWork() && !mayorAlive) return;
if (!scheduling.hasRunningAgents(this.sql) && !mayorAlive) return;

const TOKEN_REFRESH_INTERVAL_MS = 60 * 60_000;
const now = Date.now();
Expand All @@ -4671,7 +4671,7 @@ export class TownDO extends DurableObject<Env> {
*/
private async stopContainerIfIdle(): Promise<void> {
await _stopContainerIfIdle({
hasActiveWork: () => this.hasActiveWork(),
hasRunningAgents: () => scheduling.hasRunningAgents(this.sql),
isDraining: () => this._draining || this.containerStartsBlocked(),
getMayor: () => agents.listAgents(this.sql, { role: 'mayor' })[0] ?? null,
getTownId: () => this.townId,
Expand Down Expand Up @@ -5094,16 +5094,13 @@ export class TownDO extends DurableObject<Env> {
}
}

private async ensureContainerReady(
cachedRigList?: rigs.RigRecord[],
cachedActiveWork?: boolean
): Promise<void> {
private async ensureContainerReady(cachedRigList?: rigs.RigRecord[]): Promise<void> {
if (this.containerStartsBlocked()) return;
const rigList = cachedRigList ?? rigs.listRigs(this.sql);
if (rigList.length === 0) return;

const hasWork = cachedActiveWork ?? this.hasActiveWork();
if (!hasWork && !this._draining) {
// Other active work starts the container when it is dispatched.
if (!scheduling.hasRunningAgents(this.sql) && !this._draining) {
const newestRigAge = rigList.reduce((min, r) => {
const age = Date.now() - new Date(r.created_at).getTime();
return Math.min(min, age);
Expand Down
6 changes: 3 additions & 3 deletions services/gastown/src/dos/town/container-idle-stop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ function makeDeps(overrides: Partial<IdleStopDeps> = {}): TestDeps {
const events: Array<{ event: string; townId: string; reason: string; error?: string }> = [];

return {
hasActiveWork: overrides.hasActiveWork ?? (() => false),
hasRunningAgents: overrides.hasRunningAgents ?? (() => false),
isDraining: overrides.isDraining ?? (() => false),
getMayor: overrides.getMayor ?? (() => null),
getTownId: overrides.getTownId ?? (() => 'town-1'),
Expand Down Expand Up @@ -70,8 +70,8 @@ function makeDeps(overrides: Partial<IdleStopDeps> = {}): TestDeps {
}

describe('stopContainerIfIdle', () => {
it('does not stop when town has active work', async () => {
const deps = makeDeps({ hasActiveWork: () => true });
it('does not stop while an agent is running', async () => {
const deps = makeDeps({ hasRunningAgents: () => true });
await stopContainerIfIdle(deps);
expect(deps._stopFn).not.toHaveBeenCalled();
});
Expand Down
5 changes: 3 additions & 2 deletions services/gastown/src/dos/town/container-idle-stop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const CONTAINER_IDLE_STOP_THRESHOLD_MS = 5 * 60_000;
export const CONTAINER_IDLE_STOP_THROTTLE_MS = 2 * 60_000;

export type IdleStopDeps = {
hasActiveWork: () => boolean;
hasRunningAgents: () => boolean;
isDraining: () => boolean;
getMayor: () => Agent | null;
getTownId: () => string | null;
Expand All @@ -29,7 +29,7 @@ export type IdleStopDeps = {
};

export async function stopContainerIfIdle(deps: IdleStopDeps): Promise<void> {
if (deps.hasActiveWork()) return;
if (deps.hasRunningAgents()) return;
if (deps.isDraining()) return;

const mayor = deps.getMayor();
Expand Down Expand Up @@ -71,6 +71,7 @@ export async function stopContainerIfIdle(deps: IdleStopDeps): Promise<void> {
await stub.stop();
await deps.setLastIdleStopAt(now);
deps.writeEventFn({ event: 'container.idle_stop', townId, reason });
logger.info('stopContainerIfIdle: stopped idle container', { reason });
} catch (err) {
logger.warn('stopContainerIfIdle: stop() failed', {
error: err instanceof Error ? err.message : String(err),
Expand Down
45 changes: 27 additions & 18 deletions services/gastown/src/dos/town/scheduling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -262,23 +262,6 @@ export function dispatchUnblockedBeads(ctx: SchedulingContext, closedBeadId: str
* before).
*/
export function hasActiveWork(sql: SqlStorage): boolean {
// Stalled agents older than 30min no longer count as active work: they
// typically represent stuck rows (container crashed hard, /status keeps
// returning running/unknown). Keeping them in the active set would pin
// the alarm at its 5s fast cadence indefinitely. The stalled->idle
// auto-transition in reconcileAgents cleans them up after 2h 30min.
const hasActiveAgents = (): boolean =>
countOf([
...query(
sql,
/* sql */ `SELECT COUNT(*) as cnt FROM ${agent_metadata}
WHERE ${agent_metadata.status} = 'working'
OR (${agent_metadata.status} = 'stalled'
AND ${agent_metadata.last_activity_at} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-30 minutes'))`,
[]
),
]) > 0;

// Idle agents that already hold a hook — the reconciler should dispatch
// them on the next tick.
const hasHookedIdleAgents = (): boolean =>
Expand Down Expand Up @@ -333,14 +316,40 @@ export function hasActiveWork(sql: SqlStorage): boolean {
]) > 0;

return (
hasActiveAgents() ||
hasRunningAgents(sql) ||
hasHookedIdleAgents() ||
hasOpenMergeRequests() ||
hasOpenTriageBeads() ||
hasUnassignedIssues()
);
}

/**
* Returns true if an agent process should be running in the container. Only
* this keeps the container awake: other active work starts the container
* when the reconciler dispatches it, and work it never dispatches must not
* hold the container forever.
*/
export function hasRunningAgents(sql: SqlStorage): boolean {
// Stalled agents older than 30min no longer count as active work: they
// typically represent stuck rows (container crashed hard, /status keeps
// returning running/unknown). Keeping them in the active set would pin
// the alarm at its 5s fast cadence indefinitely. The stalled->idle
// auto-transition in reconcileAgents cleans them up after 2h 30min.
return (
countOf([
...query(
sql,
/* sql */ `SELECT COUNT(*) as cnt FROM ${agent_metadata}
WHERE ${agent_metadata.status} = 'working'
OR (${agent_metadata.status} = 'stalled'
AND ${agent_metadata.last_activity_at} > strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-30 minutes'))`,
[]
),
]) > 0
);
}

/** Read the `cnt` column off the first row of a `SELECT COUNT(*) as cnt` query. */
function countOf(rows: ReadonlyArray<Record<string, unknown>>): number {
return Number(rows[0]?.cnt ?? 0);
Expand Down
55 changes: 55 additions & 0 deletions services/gastown/test/integration/container-idle.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { env, runDurableObjectAlarm, runInDurableObject } from 'cloudflare:test';
import { describe, expect, it } from 'vitest';
import { getTownContainerStub } from '../../src/dos/TownContainer.do';
import { HELD_LABEL } from '../../src/dos/town/patrol';

// Older than the 5-minute window in which a new rig keeps the container warm.
const OLD_RIG_CREATED_AT = new Date(Date.now() - 60 * 60_000).toISOString();

async function townWithRunningContainer() {
const townId = `container-idle-${crypto.randomUUID()}`;
const town = env.TOWN.get(env.TOWN.idFromName(townId));
await town.setTownId(townId);
await town.addRig({
rigId: 'rig-1',
name: 'Rig',
gitUrl: 'https://github.com/org/repo.git',
defaultBranch: 'main',
});
await runInDurableObject(town, (_instance, state) => {
state.storage.sql.exec('UPDATE rigs SET created_at = ?', OLD_RIG_CREATED_AT);
return state.storage.setAlarm(Date.now() + 60_000);
});
const container = getTownContainerStub(env, townId);
await container.warmUp();
return { townId, town, container };
}

describe('Town alarm idle container stop', () => {
it('stops the container when the town only has work it never dispatches', async () => {
const { town, container } = await townWithRunningContainer();
// A held issue counts as active work, but the reconciler never dispatches it.
await town.createBead({ type: 'issue', title: 'Held', rig_id: 'rig-1', labels: [HELD_LABEL] });

expect(await runDurableObjectAlarm(town)).toBe(true);

expect(await container.getState()).toEqual({ status: 'stopped' });
});

it('keeps the container while an agent is working', async () => {
const { townId, town, container } = await townWithRunningContainer();
const agent = await town.registerAgent({
role: 'polecat',
name: 'P1',
identity: `polecat-${townId}`,
});
const bead = await town.createBead({ type: 'issue', title: 'Work', rig_id: 'rig-1' });
await town.hookBead(agent.id, bead.bead_id);
await town.updateAgentStatus(agent.id, 'working');
await town.touchAgentHeartbeat(agent.id);

expect(await runDurableObjectAlarm(town)).toBe(true);

expect(await container.getState()).toEqual({ status: 'healthy' });
});
});