diff --git a/packages/core/src/__tests__/recovery-actions.test.ts b/packages/core/src/__tests__/recovery-actions.test.ts index 546262dd6..c645bcfab 100644 --- a/packages/core/src/__tests__/recovery-actions.test.ts +++ b/packages/core/src/__tests__/recovery-actions.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { readMetadataRaw } from "../metadata.js"; import { getSessionsDir } from "../paths.js"; -import { escalateSession, recoverSession } from "../recovery/actions.js"; +import { cleanupSession, escalateSession, recoverSession } from "../recovery/actions.js"; import { runRecovery } from "../recovery/manager.js"; import { getRecoveryLogPath, scanAllSessions } from "../recovery/scanner.js"; import { @@ -13,7 +13,7 @@ import { type RecoveryAssessment, type RecoveryContext, } from "../recovery/types.js"; -import type { OrchestratorConfig, PluginRegistry } from "../types.js"; +import type { OrchestratorConfig, PluginRegistry, Runtime, Workspace } from "../types.js"; function makeConfig(rootDir: string): OrchestratorConfig { return { @@ -242,6 +242,118 @@ describe("escalateSession", () => { }); }); +describe("cleanupSession", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("continues cleanup and calls deleteMetadata even when workspace.destroy throws", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const workspacePath = join(rootDir, "worktree"); + const mockWorkspace: Workspace = { + name: "worktree", + create: vi.fn(), + destroy: vi.fn().mockRejectedValue(new Error("Workspace destroy failed")), + list: vi.fn(), + exists: vi.fn(), + }; + const registry: PluginRegistry = { + register: vi.fn(), + get: vi.fn().mockImplementation((slot: string) => { + if (slot === "workspace") return mockWorkspace; + return null; + }), + list: vi.fn().mockReturnValue([]), + loadBuiltins: vi.fn().mockResolvedValue(undefined), + loadFromConfig: vi.fn().mockResolvedValue(undefined), + }; + const assessment = makeAssessment({ + action: "cleanup", + classification: "dead", + runtimeAlive: false, + workspaceExists: true, + workspacePath, + rawMetadata: { + ...makeAssessment().rawMetadata, + worktree: workspacePath, + }, + }); + const context = makeContext(rootDir); + + const result = await cleanupSession(assessment, config, registry, context); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + + expect(mockWorkspace.destroy).toHaveBeenCalled(); + expect(result.success).toBe(true); + expect(existsSync(join(sessionsDir, "app-1"))).toBe(false); + }); + + it("continues cleanup and calls workspace.destroy and deleteMetadata even when runtime.destroy throws", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const workspacePath = join(rootDir, "worktree"); + const mockRuntime: Runtime = { + name: "tmux", + create: vi.fn(), + destroy: vi.fn().mockRejectedValue(new Error("Runtime destroy failed")), + sendMessage: vi.fn(), + getOutput: vi.fn(), + isAlive: vi.fn(), + }; + const mockWorkspace: Workspace = { + name: "worktree", + create: vi.fn(), + destroy: vi.fn().mockResolvedValue(undefined), + list: vi.fn(), + exists: vi.fn(), + }; + const registry: PluginRegistry = { + register: vi.fn(), + get: vi.fn().mockImplementation((slot: string) => { + if (slot === "runtime") return mockRuntime; + if (slot === "workspace") return mockWorkspace; + return null; + }), + list: vi.fn().mockReturnValue([]), + loadBuiltins: vi.fn().mockResolvedValue(undefined), + loadFromConfig: vi.fn().mockResolvedValue(undefined), + }; + const assessment = makeAssessment({ + action: "cleanup", + classification: "partial", + runtimeAlive: true, + workspaceExists: true, + workspacePath, + rawMetadata: { + ...makeAssessment().rawMetadata, + worktree: workspacePath, + }, + }); + const context = makeContext(rootDir); + + const result = await cleanupSession(assessment, config, registry, context); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + + expect(mockRuntime.destroy).toHaveBeenCalled(); + expect(mockWorkspace.destroy).toHaveBeenCalled(); + expect(result.success).toBe(true); + expect(existsSync(join(sessionsDir, "app-1"))).toBe(false); + }); +}); + describe("recovery manager and scanner", () => { let rootDir: string; diff --git a/packages/core/src/__tests__/recovery-validator.test.ts b/packages/core/src/__tests__/recovery-validator.test.ts index fa585db6d..cab682a4f 100644 --- a/packages/core/src/__tests__/recovery-validator.test.ts +++ b/packages/core/src/__tests__/recovery-validator.test.ts @@ -121,4 +121,94 @@ describe("recovery validator", () => { expect(mockOrchestratorAgent.isProcessRunning).toHaveBeenCalled(); expect(mockWorkerAgent.isProcessRunning).not.toHaveBeenCalled(); }); + + it("sets runtimeAlive to false when runtime.isAlive throws an error", async () => { + rootDir = join(tmpdir(), `ao-recovery-validator-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + const projectPath = join(rootDir, "project"); + mkdirSync(projectPath, { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const mockRuntime: Runtime = { + name: "tmux", + create: vi.fn(), + destroy: vi.fn(), + sendMessage: vi.fn(), + getOutput: vi.fn(), + isAlive: vi.fn().mockRejectedValue(new Error("Runtime check failed")), + }; + const mockWorkspace: Workspace = { + name: "worktree", + create: vi.fn(), + destroy: vi.fn(), + list: vi.fn(), + exists: vi.fn().mockResolvedValue(true), + }; + const mockAgent: Agent = { + name: "mock-agent", + processName: "mock-agent", + getLaunchCommand: vi.fn(), + getEnvironment: vi.fn(), + detectActivity: vi.fn(), + getActivityState: vi.fn(), + isProcessRunning: vi.fn().mockResolvedValue(false), + getSessionInfo: vi.fn(), + }; + const registry: PluginRegistry = { + register: vi.fn(), + get: vi.fn().mockImplementation((slot: string) => { + if (slot === "runtime") return mockRuntime; + if (slot === "workspace") return mockWorkspace; + if (slot === "agent") return mockAgent; + return null; + }), + list: vi.fn().mockReturnValue([]), + loadBuiltins: vi.fn().mockResolvedValue(undefined), + loadFromConfig: vi.fn().mockResolvedValue(undefined), + }; + const config: OrchestratorConfig = { + configPath: join(rootDir, "agent-orchestrator.yaml"), + port: 3000, + readyThresholdMs: 300_000, + defaults: { + runtime: "tmux", + agent: "mock-agent", + workspace: "worktree", + notifiers: ["desktop"], + }, + projects: { + app: { + name: "app", + repo: "org/repo", + path: projectPath, + defaultBranch: "main", + sessionPrefix: "app", + }, + }, + notifiers: {}, + notificationRouting: { + urgent: ["desktop"], + action: ["desktop"], + warning: [], + info: [], + }, + reactions: {}, + }; + const scanned: ScannedSession = { + sessionId: "app-1", + projectId: "app", + project: config.projects.app, + sessionsDir: getSessionsDir(config.configPath, projectPath), + rawMetadata: { + worktree: projectPath, + status: "working", + runtimeHandle: JSON.stringify({ id: "rt-1", runtimeName: "tmux", data: {} }), + }, + }; + + const assessment = await validateSession(scanned, config, registry); + + expect(mockRuntime.isAlive).toHaveBeenCalled(); + expect(assessment.runtimeAlive).toBe(false); + }); });