fix: honor live startup evidence in recovery (#95)
This commit is contained in:
parent
f2f1a4ddb6
commit
f4fdaa7f3f
|
|
@ -382,6 +382,169 @@ describe("recovery validator", () => {
|
|||
expect(assessment.action).toBe("skip");
|
||||
});
|
||||
|
||||
it("recovers healthy sessions even if stale metadata still says detecting", async () => {
|
||||
rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`);
|
||||
mkdirSync(rootDir, { recursive: true });
|
||||
const projectPath = join(rootDir, "project");
|
||||
mkdirSync(projectPath, { recursive: true });
|
||||
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
|
||||
|
||||
const mockRuntime: Runtime = {
|
||||
name: "tmux",
|
||||
create: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
sendMessage: vi.fn(),
|
||||
getOutput: vi.fn(),
|
||||
isAlive: vi.fn().mockResolvedValue(true),
|
||||
};
|
||||
const mockWorkspace: Workspace = {
|
||||
name: "worktree",
|
||||
create: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
list: vi.fn(),
|
||||
exists: vi.fn().mockResolvedValue(true),
|
||||
};
|
||||
const mockAgent: Agent = {
|
||||
name: "mock-agent",
|
||||
processName: "mock-agent",
|
||||
getLaunchCommand: vi.fn(),
|
||||
getEnvironment: vi.fn(),
|
||||
detectActivity: vi.fn(),
|
||||
getActivityState: vi.fn(),
|
||||
isProcessRunning: vi.fn().mockResolvedValue(true),
|
||||
getSessionInfo: vi.fn(),
|
||||
};
|
||||
const registry: PluginRegistry = {
|
||||
register: vi.fn(),
|
||||
get: vi.fn().mockImplementation((slot: string) => {
|
||||
if (slot === "runtime") return mockRuntime;
|
||||
if (slot === "workspace") return mockWorkspace;
|
||||
if (slot === "agent") return mockAgent;
|
||||
return null;
|
||||
}),
|
||||
list: vi.fn().mockReturnValue([]),
|
||||
loadBuiltins: vi.fn().mockResolvedValue(undefined),
|
||||
loadFromConfig: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const config: OrchestratorConfig = {
|
||||
configPath: join(rootDir, "agent-orchestrator.yaml"),
|
||||
port: 3000,
|
||||
readyThresholdMs: 300_000,
|
||||
power: { preventIdleSleep: false },
|
||||
defaults: {
|
||||
runtime: "tmux",
|
||||
agent: "mock-agent",
|
||||
workspace: "worktree",
|
||||
notifiers: ["desktop"],
|
||||
},
|
||||
projects: {
|
||||
app: {
|
||||
name: "app",
|
||||
repo: "org/repo",
|
||||
path: projectPath,
|
||||
defaultBranch: "main",
|
||||
sessionPrefix: "app",
|
||||
},
|
||||
},
|
||||
notifiers: {},
|
||||
notificationRouting: {
|
||||
urgent: ["desktop"],
|
||||
action: ["desktop"],
|
||||
warning: [],
|
||||
info: [],
|
||||
},
|
||||
reactions: {},
|
||||
};
|
||||
const scanned: ScannedSession = {
|
||||
sessionId: "app-1",
|
||||
projectId: "app",
|
||||
project: config.projects.app,
|
||||
sessionsDir: getSessionsDir(config.configPath, projectPath),
|
||||
rawMetadata: {
|
||||
worktree: projectPath,
|
||||
status: "detecting",
|
||||
runtimeHandle: JSON.stringify({ id: "rt-1", runtimeName: "tmux", data: {} }),
|
||||
},
|
||||
};
|
||||
|
||||
const assessment = await validateSession(scanned, config, registry);
|
||||
|
||||
expect(assessment.classification).toBe("live");
|
||||
expect(assessment.recoveryRule).toBe("auto");
|
||||
expect(assessment.action).toBe("recover");
|
||||
});
|
||||
|
||||
it("treats missing runtime handle plus missing workspace as dead metadata", async () => {
|
||||
rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`);
|
||||
mkdirSync(rootDir, { recursive: true });
|
||||
const projectPath = join(rootDir, "project");
|
||||
mkdirSync(projectPath, { recursive: true });
|
||||
writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8");
|
||||
|
||||
const mockWorkspace: Workspace = {
|
||||
name: "worktree",
|
||||
create: vi.fn(),
|
||||
destroy: vi.fn(),
|
||||
list: vi.fn(),
|
||||
exists: vi.fn().mockResolvedValue(false),
|
||||
};
|
||||
const registry: PluginRegistry = {
|
||||
register: vi.fn(),
|
||||
get: vi.fn().mockImplementation((slot: string) => {
|
||||
if (slot === "workspace") return mockWorkspace;
|
||||
return null;
|
||||
}),
|
||||
list: vi.fn().mockReturnValue([]),
|
||||
loadBuiltins: vi.fn().mockResolvedValue(undefined),
|
||||
loadFromConfig: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const config: OrchestratorConfig = {
|
||||
configPath: join(rootDir, "agent-orchestrator.yaml"),
|
||||
port: 3000,
|
||||
readyThresholdMs: 300_000,
|
||||
power: { preventIdleSleep: false },
|
||||
defaults: {
|
||||
runtime: "tmux",
|
||||
agent: "mock-agent",
|
||||
workspace: "worktree",
|
||||
notifiers: ["desktop"],
|
||||
},
|
||||
projects: {
|
||||
app: {
|
||||
name: "app",
|
||||
repo: "org/repo",
|
||||
path: projectPath,
|
||||
defaultBranch: "main",
|
||||
sessionPrefix: "app",
|
||||
},
|
||||
},
|
||||
notifiers: {},
|
||||
notificationRouting: {
|
||||
urgent: ["desktop"],
|
||||
action: ["desktop"],
|
||||
warning: [],
|
||||
info: [],
|
||||
},
|
||||
reactions: {},
|
||||
};
|
||||
const scanned: ScannedSession = {
|
||||
sessionId: "app-1",
|
||||
projectId: "app",
|
||||
project: config.projects.app,
|
||||
sessionsDir: getSessionsDir(config.configPath, projectPath),
|
||||
rawMetadata: {
|
||||
worktree: join(rootDir, "missing-worktree"),
|
||||
status: "working",
|
||||
},
|
||||
};
|
||||
|
||||
const assessment = await validateSession(scanned, config, registry);
|
||||
|
||||
expect(assessment.classification).toBe("dead");
|
||||
expect(assessment.recoveryRule).toBe("auto");
|
||||
expect(assessment.action).toBe("cleanup");
|
||||
});
|
||||
|
||||
it("respects escalatePartial=false for non-disagreement partial sessions", async () => {
|
||||
rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`);
|
||||
mkdirSync(rootDir, { recursive: true });
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ export async function validateSession(
|
|||
metadataStatus,
|
||||
runtimeProbeSucceeded,
|
||||
processProbeSucceeded,
|
||||
runtimeHandle !== null,
|
||||
);
|
||||
const signalDisagreement =
|
||||
runtimeProbeSucceeded &&
|
||||
|
|
@ -155,23 +156,24 @@ function classifySession(
|
|||
metadataStatus: SessionStatus,
|
||||
runtimeProbeSucceeded: boolean,
|
||||
processProbeSucceeded: boolean,
|
||||
hasRuntimeHandle: boolean,
|
||||
): RecoveryClassification {
|
||||
if (TERMINAL_STATUSES_SET.has(metadataStatus)) {
|
||||
return "unrecoverable";
|
||||
}
|
||||
|
||||
if (metadataStatus === "detecting" || !runtimeProbeSucceeded || !processProbeSucceeded) {
|
||||
return "partial";
|
||||
}
|
||||
|
||||
if (runtimeAlive && workspaceExists && agentProcessRunning) {
|
||||
return "live";
|
||||
}
|
||||
|
||||
if (!runtimeAlive && !workspaceExists) {
|
||||
if (!workspaceExists && (!hasRuntimeHandle || (runtimeProbeSucceeded && !runtimeAlive))) {
|
||||
return "dead";
|
||||
}
|
||||
|
||||
if (metadataStatus === "detecting" || !runtimeProbeSucceeded || !processProbeSucceeded) {
|
||||
return "partial";
|
||||
}
|
||||
|
||||
if (runtimeAlive && !workspaceExists) {
|
||||
return "partial";
|
||||
}
|
||||
|
|
@ -194,15 +196,18 @@ function determineRecoveryRule(
|
|||
recoveryConfig: RecoveryConfig = DEFAULT_RECOVERY_CONFIG,
|
||||
): "auto" | "human" | "skip" {
|
||||
if (classification === "unrecoverable") return "skip";
|
||||
if (metadataStatus === "detecting" || signalDisagreement) {
|
||||
if (signalDisagreement) {
|
||||
return "human";
|
||||
}
|
||||
if (classification === "live" || classification === "dead") {
|
||||
return "auto";
|
||||
}
|
||||
if (metadataStatus === "detecting") {
|
||||
return "human";
|
||||
}
|
||||
if (classification === "partial") {
|
||||
return recoveryConfig.escalatePartial ? "human" : "auto";
|
||||
}
|
||||
if (classification === "live" || classification === "dead") {
|
||||
return "auto";
|
||||
}
|
||||
return "human";
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue