fix: honor live startup evidence in recovery (#95)

This commit is contained in:
harshitsinghbhandari 2026-04-16 23:06:49 +05:30
parent f2f1a4ddb6
commit f4fdaa7f3f
2 changed files with 177 additions and 9 deletions

View File

@ -382,6 +382,169 @@ describe("recovery validator", () => {
expect(assessment.action).toBe("skip");
});
it("recovers healthy sessions even if stale metadata still says detecting", async () => {
rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
const projectPath = join(rootDir, "project");
mkdirSync(projectPath, { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const mockRuntime: Runtime = {
name: "tmux",
create: vi.fn(),
destroy: vi.fn(),
sendMessage: vi.fn(),
getOutput: vi.fn(),
isAlive: vi.fn().mockResolvedValue(true),
};
const mockWorkspace: Workspace = {
name: "worktree",
create: vi.fn(),
destroy: vi.fn(),
list: vi.fn(),
exists: vi.fn().mockResolvedValue(true),
};
const mockAgent: Agent = {
name: "mock-agent",
processName: "mock-agent",
getLaunchCommand: vi.fn(),
getEnvironment: vi.fn(),
detectActivity: vi.fn(),
getActivityState: vi.fn(),
isProcessRunning: vi.fn().mockResolvedValue(true),
getSessionInfo: vi.fn(),
};
const registry: PluginRegistry = {
register: vi.fn(),
get: vi.fn().mockImplementation((slot: string) => {
if (slot === "runtime") return mockRuntime;
if (slot === "workspace") return mockWorkspace;
if (slot === "agent") return mockAgent;
return null;
}),
list: vi.fn().mockReturnValue([]),
loadBuiltins: vi.fn().mockResolvedValue(undefined),
loadFromConfig: vi.fn().mockResolvedValue(undefined),
};
const config: OrchestratorConfig = {
configPath: join(rootDir, "agent-orchestrator.yaml"),
port: 3000,
readyThresholdMs: 300_000,
power: { preventIdleSleep: false },
defaults: {
runtime: "tmux",
agent: "mock-agent",
workspace: "worktree",
notifiers: ["desktop"],
},
projects: {
app: {
name: "app",
repo: "org/repo",
path: projectPath,
defaultBranch: "main",
sessionPrefix: "app",
},
},
notifiers: {},
notificationRouting: {
urgent: ["desktop"],
action: ["desktop"],
warning: [],
info: [],
},
reactions: {},
};
const scanned: ScannedSession = {
sessionId: "app-1",
projectId: "app",
project: config.projects.app,
sessionsDir: getSessionsDir(config.configPath, projectPath),
rawMetadata: {
worktree: projectPath,
status: "detecting",
runtimeHandle: JSON.stringify({ id: "rt-1", runtimeName: "tmux", data: {} }),
},
};
const assessment = await validateSession(scanned, config, registry);
expect(assessment.classification).toBe("live");
expect(assessment.recoveryRule).toBe("auto");
expect(assessment.action).toBe("recover");
});
it("treats missing runtime handle plus missing workspace as dead metadata", async () => {
rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });
const projectPath = join(rootDir, "project");
mkdirSync(projectPath, { recursive: true });
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
const mockWorkspace: Workspace = {
name: "worktree",
create: vi.fn(),
destroy: vi.fn(),
list: vi.fn(),
exists: vi.fn().mockResolvedValue(false),
};
const registry: PluginRegistry = {
register: vi.fn(),
get: vi.fn().mockImplementation((slot: string) => {
if (slot === "workspace") return mockWorkspace;
return null;
}),
list: vi.fn().mockReturnValue([]),
loadBuiltins: vi.fn().mockResolvedValue(undefined),
loadFromConfig: vi.fn().mockResolvedValue(undefined),
};
const config: OrchestratorConfig = {
configPath: join(rootDir, "agent-orchestrator.yaml"),
port: 3000,
readyThresholdMs: 300_000,
power: { preventIdleSleep: false },
defaults: {
runtime: "tmux",
agent: "mock-agent",
workspace: "worktree",
notifiers: ["desktop"],
},
projects: {
app: {
name: "app",
repo: "org/repo",
path: projectPath,
defaultBranch: "main",
sessionPrefix: "app",
},
},
notifiers: {},
notificationRouting: {
urgent: ["desktop"],
action: ["desktop"],
warning: [],
info: [],
},
reactions: {},
};
const scanned: ScannedSession = {
sessionId: "app-1",
projectId: "app",
project: config.projects.app,
sessionsDir: getSessionsDir(config.configPath, projectPath),
rawMetadata: {
worktree: join(rootDir, "missing-worktree"),
status: "working",
},
};
const assessment = await validateSession(scanned, config, registry);
expect(assessment.classification).toBe("dead");
expect(assessment.recoveryRule).toBe("auto");
expect(assessment.action).toBe("cleanup");
});
it("respects escalatePartial=false for non-disagreement partial sessions", async () => {
rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`);
mkdirSync(rootDir, { recursive: true });

View File

@ -105,6 +105,7 @@ export async function validateSession(
metadataStatus,
runtimeProbeSucceeded,
processProbeSucceeded,
runtimeHandle !== null,
);
const signalDisagreement =
runtimeProbeSucceeded &&
@ -155,23 +156,24 @@ function classifySession(
metadataStatus: SessionStatus,
runtimeProbeSucceeded: boolean,
processProbeSucceeded: boolean,
hasRuntimeHandle: boolean,
): RecoveryClassification {
if (TERMINAL_STATUSES_SET.has(metadataStatus)) {
return "unrecoverable";
}
if (metadataStatus === "detecting" || !runtimeProbeSucceeded || !processProbeSucceeded) {
return "partial";
}
if (runtimeAlive && workspaceExists && agentProcessRunning) {
return "live";
}
if (!runtimeAlive && !workspaceExists) {
if (!workspaceExists && (!hasRuntimeHandle || (runtimeProbeSucceeded && !runtimeAlive))) {
return "dead";
}
if (metadataStatus === "detecting" || !runtimeProbeSucceeded || !processProbeSucceeded) {
return "partial";
}
if (runtimeAlive && !workspaceExists) {
return "partial";
}
@ -194,15 +196,18 @@ function determineRecoveryRule(
recoveryConfig: RecoveryConfig = DEFAULT_RECOVERY_CONFIG,
): "auto" | "human" | "skip" {
if (classification === "unrecoverable") return "skip";
if (metadataStatus === "detecting" || signalDisagreement) {
if (signalDisagreement) {
return "human";
}
if (classification === "live" || classification === "dead") {
return "auto";
}
if (metadataStatus === "detecting") {
return "human";
}
if (classification === "partial") {
return recoveryConfig.escalatePartial ? "human" : "auto";
}
if (classification === "live" || classification === "dead") {
return "auto";
}
return "human";
}