diff --git a/packages/core/src/__tests__/recovery-actions.test.ts b/packages/core/src/__tests__/recovery-actions.test.ts new file mode 100644 index 000000000..1f9cfc392 --- /dev/null +++ b/packages/core/src/__tests__/recovery-actions.test.ts @@ -0,0 +1,284 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { readMetadataRaw } from "../metadata.js"; +import { getSessionsDir } from "../paths.js"; +import { escalateSession, recoverSession } from "../recovery/actions.js"; +import { runRecovery } from "../recovery/manager.js"; +import { getRecoveryLogPath, scanAllSessions } from "../recovery/scanner.js"; +import { + DEFAULT_RECOVERY_CONFIG, + type RecoveryAssessment, + type RecoveryContext, +} from "../recovery/types.js"; +import type { OrchestratorConfig, PluginRegistry } from "../types.js"; + +function makeConfig(rootDir: string): OrchestratorConfig { + return { + configPath: join(rootDir, "agent-orchestrator.yaml"), + port: 3000, + readyThresholdMs: 300_000, + defaults: { + runtime: "tmux", + agent: "claude-code", + workspace: "worktree", + notifiers: ["desktop"], + }, + projects: { + app: { + name: "app", + repo: "org/repo", + path: join(rootDir, "project"), + defaultBranch: "main", + sessionPrefix: "app", + }, + }, + notifiers: {}, + notificationRouting: { + urgent: ["desktop"], + action: ["desktop"], + warning: ["desktop"], + info: ["desktop"], + }, + reactions: {}, + }; +} + +function makeRegistry(): PluginRegistry { + return { + register: vi.fn(), + get: vi.fn().mockReturnValue(null), + list: vi.fn().mockReturnValue([]), + loadBuiltins: vi.fn().mockResolvedValue(undefined), + loadFromConfig: vi.fn().mockResolvedValue(undefined), + }; +} + +function makeAssessment(overrides: Partial = {}): RecoveryAssessment { + return { + sessionId: "app-1", + projectId: "app", + classification: "live", + action: "recover", + reason: "Session is running normally", + runtimeAlive: true, + runtimeHandle: { id: "rt-1", runtimeName: "tmux", data: {} }, + workspaceExists: true, + workspacePath: "/tmp/worktree", + agentProcessRunning: true, + agentActivity: "active", + metadataValid: true, + metadataStatus: "working", + rawMetadata: { + project: "app", + branch: "feat/test", + issue: "123", + pr: "https://github.com/org/repo/pull/42", + createdAt: "2025-01-01T00:00:00.000Z", + status: "working", + summary: "Recovered summary", + }, + ...overrides, + }; +} + +function makeContext(rootDir: string, overrides: Partial = {}): RecoveryContext { + return { + configPath: join(rootDir, "agent-orchestrator.yaml"), + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: join(rootDir, "recovery.log"), + }, + dryRun: false, + ...overrides, + }; +} + +describe("recoverSession", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("persists restoredAt and returns a session with restoredAt", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment(); + const context = makeContext(rootDir); + + const result = await recoverSession(assessment, config, registry, context); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + const metadata = readMetadataRaw(sessionsDir, assessment.sessionId); + + expect(result.success).toBe(true); + expect(result.session?.restoredAt).toBeInstanceOf(Date); + expect(metadata?.["restoredAt"]).toBeDefined(); + expect(metadata?.["recoveredAt"]).toBeUndefined(); + }); + + it("returns the max-attempt reason when recovery escalates", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment({ + rawMetadata: { + ...makeAssessment().rawMetadata, + recoveryCount: "3", + }, + }); + const context = makeContext(rootDir, { + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: join(rootDir, "recovery.log"), + maxRecoveryAttempts: 3, + }, + }); + + const result = await recoverSession(assessment, config, registry, context); + + expect(result.success).toBe(true); + expect(result.action).toBe("escalate"); + expect(result.reason).toBe("Exceeded max recovery attempts (3)"); + }); + + it("dry-run recovery reports escalate when attempts exceed limit", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment({ + rawMetadata: { + ...makeAssessment().rawMetadata, + recoveryCount: "3", + }, + }); + const context = makeContext(rootDir, { + dryRun: true, + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: join(rootDir, "recovery.log"), + maxRecoveryAttempts: 3, + }, + }); + + const result = await recoverSession(assessment, config, registry, context); + + expect(result.success).toBe(true); + expect(result.action).toBe("escalate"); + expect(result.requiresManualIntervention).toBe(true); + expect(result.reason).toBe("Exceeded max recovery attempts (3)"); + }); +}); + +describe("escalateSession", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("uses the assessment reason during dry runs", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment({ + action: "escalate", + classification: "partial", + reason: "Workspace exists but runtime is missing", + }); + const context = makeContext(rootDir, { dryRun: true }); + + const result = await escalateSession(assessment, config, registry, context); + + expect(result.success).toBe(true); + expect(result.action).toBe("escalate"); + expect(result.reason).toBe("Workspace exists but runtime is missing"); + }); +}); + +describe("recovery manager and scanner", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("respects custom recovery logPath in manager options", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + mkdirSync(sessionsDir, { recursive: true }); + writeFileSync( + join(sessionsDir, "app-1"), + "project=app\nstatus=terminated\nworktree=/tmp/worktree\n", + "utf-8", + ); + + const customLogPath = join(rootDir, "custom-recovery.log"); + const registry = makeRegistry(); + + await runRecovery({ + config, + registry, + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: customLogPath, + }, + }); + + expect(existsSync(customLogPath)).toBe(true); + expect(readFileSync(customLogPath, "utf-8")).toContain('"sessionId":"app-1"'); + + const defaultLogPath = getRecoveryLogPath(config.configPath); + expect(defaultLogPath).not.toBe(customLogPath); + }); + + it("scans sessions using metadata listing rules", () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + mkdirSync(sessionsDir, { recursive: true }); + + writeFileSync(join(sessionsDir, "app-1"), "project=app\nstatus=working\n", "utf-8"); + writeFileSync(join(sessionsDir, ".tmp"), "project=app\n", "utf-8"); + writeFileSync(join(sessionsDir, "bad.session"), "project=app\n", "utf-8"); + + const scanned = scanAllSessions(config); + + expect(scanned).toHaveLength(1); + expect(scanned[0]?.sessionId).toBe("app-1"); + }); +}); diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index 2c22d6007..341040f39 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { isRetryableHttpStatus, normalizeRetryConfig, readLastJsonlEntry } from "../utils.js"; +import { parsePrFromUrl } from "../utils/pr.js"; describe("readLastJsonlEntry", () => { let tmpDir: string; @@ -114,3 +115,27 @@ describe("retry utilities", () => { }); }); }); + +describe("parsePrFromUrl", () => { + it("parses GitHub PR URLs", () => { + expect(parsePrFromUrl("https://github.com/foo/bar/pull/123")).toEqual({ + owner: "foo", + repo: "bar", + number: 123, + url: "https://github.com/foo/bar/pull/123", + }); + }); + + it("falls back to trailing number for non-GitHub URLs", () => { + expect(parsePrFromUrl("https://gitlab.com/foo/bar/-/merge_requests/456")).toEqual({ + owner: "", + repo: "", + number: 456, + url: "https://gitlab.com/foo/bar/-/merge_requests/456", + }); + }); + + it("returns null when the URL has no PR number", () => { + expect(parsePrFromUrl("https://example.com/foo/bar/pull/not-a-number")).toBeNull(); + }); +}); diff --git a/packages/core/src/recovery/actions.ts b/packages/core/src/recovery/actions.ts new file mode 100644 index 000000000..11d4bb762 --- /dev/null +++ b/packages/core/src/recovery/actions.ts @@ -0,0 +1,229 @@ +import type { OrchestratorConfig, PluginRegistry, Runtime, Workspace } from "../types.js"; +import { updateMetadata, deleteMetadata } from "../metadata.js"; +import { getSessionsDir } from "../paths.js"; +import { validateStatus } from "../utils/validation.js"; +import { sessionFromMetadata } from "../utils/session-from-metadata.js"; +import type { RecoveryAssessment, RecoveryResult, RecoveryContext } from "./types.js"; + +export async function recoverSession( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + registry: PluginRegistry, + context: RecoveryContext, +): Promise { + const { sessionId, projectId, rawMetadata } = assessment; + const recoveryCount = rawMetadata["recoveryCount"] + ? parseInt(rawMetadata["recoveryCount"], 10) + 1 + : 1; + + if (context.dryRun) { + if (recoveryCount > context.recoveryConfig.maxRecoveryAttempts) { + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`, + }; + } + + return { + success: true, + sessionId, + action: "recover", + }; + } + + try { + const now = new Date().toISOString(); + const preservedStatus = validateStatus(rawMetadata["status"]); + + const project = config.projects[projectId]; + const sessionsDir = getSessionsDir(config.configPath, project.path); + + if (recoveryCount > context.recoveryConfig.maxRecoveryAttempts) { + updateMetadata(sessionsDir, sessionId, { + status: "stuck", + escalatedAt: now, + escalationReason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`, + recoveryCount: String(recoveryCount), + }); + + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`, + }; + } + + updateMetadata(sessionsDir, sessionId, { + status: preservedStatus, + restoredAt: now, + recoveryCount: String(recoveryCount), + }); + + const updatedMetadata = { + ...rawMetadata, + status: preservedStatus, + restoredAt: now, + recoveryCount: String(recoveryCount), + }; + + const session = sessionFromMetadata(sessionId, updatedMetadata, { + status: preservedStatus, + runtimeHandle: assessment.runtimeHandle, + lastActivityAt: new Date(), + restoredAt: new Date(now), + }); + + return { + success: true, + sessionId, + action: "recover", + session, + }; + } catch (error) { + return { + success: false, + sessionId, + action: "recover", + error: error instanceof Error ? error.message : String(error), + }; + } +} + +export async function cleanupSession( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + registry: PluginRegistry, + context: RecoveryContext, +): Promise { + const { sessionId, projectId, rawMetadata, runtimeAlive, workspaceExists } = assessment; + + if (context.dryRun) { + return { + success: true, + sessionId, + action: "cleanup", + }; + } + + try { + const project = config.projects[projectId]; + const runtimeName = project.runtime ?? config.defaults.runtime; + const workspaceName = project.workspace ?? config.defaults.workspace; + const runtime = registry.get("runtime", runtimeName); + const workspace = registry.get("workspace", workspaceName); + + if (runtimeAlive && assessment.runtimeHandle && runtime) { + try { + await runtime.destroy(assessment.runtimeHandle); + } catch { + // ignore cleanup errors + } + } + + const workspacePath = rawMetadata["worktree"]; + if (workspacePath && workspaceExists && workspace) { + try { + await workspace.destroy(workspacePath); + } catch { + // ignore cleanup errors + } + } + + const sessionsDir = getSessionsDir(config.configPath, project.path); + + updateMetadata(sessionsDir, sessionId, { + status: "terminated", + terminatedAt: new Date().toISOString(), + terminationReason: "cleanup", + }); + + deleteMetadata(sessionsDir, sessionId, true); + + return { + success: true, + sessionId, + action: "cleanup", + }; + } catch (error) { + return { + success: false, + sessionId, + action: "cleanup", + error: error instanceof Error ? error.message : String(error), + }; + } +} + +export async function escalateSession( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + _registry: PluginRegistry, + context: RecoveryContext, +): Promise { + const { sessionId, projectId, reason } = assessment; + + if (context.dryRun) { + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason, + }; + } + + try { + const project = config.projects[projectId]; + const sessionsDir = getSessionsDir(config.configPath, project.path); + + updateMetadata(sessionsDir, sessionId, { + status: "stuck", + escalatedAt: new Date().toISOString(), + escalationReason: reason, + }); + + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason, + }; + } catch (error) { + return { + success: false, + sessionId, + action: "escalate", + error: error instanceof Error ? error.message : String(error), + requiresManualIntervention: true, + }; + } +} + +export async function executeAction( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + registry: PluginRegistry, + context: RecoveryContext, +): Promise { + switch (assessment.action) { + case "recover": + return recoverSession(assessment, config, registry, context); + case "cleanup": + return cleanupSession(assessment, config, registry, context); + case "escalate": + return escalateSession(assessment, config, registry, context); + case "skip": + default: + return { + success: true, + sessionId: assessment.sessionId, + action: "skip", + }; + } +} diff --git a/packages/core/src/recovery/index.ts b/packages/core/src/recovery/index.ts new file mode 100644 index 000000000..40dd5933f --- /dev/null +++ b/packages/core/src/recovery/index.ts @@ -0,0 +1,32 @@ +export type { + RecoveryClassification, + RecoveryAction, + RecoveryAssessment, + RecoveryResult, + RecoveryReport, + RecoveryLogEntry, + RecoveryConfig, + RecoveryContext, +} from "./types.js"; + +export { DEFAULT_RECOVERY_CONFIG } from "./types.js"; + +export { scanAllSessions, getRecoveryLogPath, type ScannedSession } from "./scanner.js"; + +export { validateSession, classifySession, determineAction } from "./validator.js"; + +export { recoverSession, cleanupSession, escalateSession, executeAction } from "./actions.js"; + +export { + writeRecoveryLog, + createLogEntry, + formatRecoveryReport, + createEmptyReport, +} from "./logger.js"; + +export { + runRecovery, + recoverSessionById, + type RecoveryManagerOptions, + type RecoveryRunResult, +} from "./manager.js"; diff --git a/packages/core/src/recovery/logger.ts b/packages/core/src/recovery/logger.ts new file mode 100644 index 000000000..0f9bfc4db --- /dev/null +++ b/packages/core/src/recovery/logger.ts @@ -0,0 +1,78 @@ +import { appendFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import type { SessionId, SessionStatus } from "../types.js"; +import type { RecoveryLogEntry, RecoveryReport } from "./types.js"; + +export function writeRecoveryLog(logPath: string, entry: RecoveryLogEntry): void { + const dir = dirname(logPath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + const line = JSON.stringify(entry) + "\n"; + appendFileSync(logPath, line, "utf-8"); +} + +export function createLogEntry( + sessionId: SessionId, + action: RecoveryLogEntry["action"], + options?: { + previousStatus?: SessionStatus; + reason?: string; + error?: string; + details?: Record; + }, +): RecoveryLogEntry { + return { + timestamp: new Date().toISOString(), + sessionId, + action, + ...options, + }; +} + +export function formatRecoveryReport(report: RecoveryReport): string { + const lines: string[] = [ + `Recovery Report - ${report.timestamp.toISOString()}`, + `Duration: ${report.durationMs}ms`, + `Sessions scanned: ${report.totalScanned}`, + "", + ]; + + if (report.recovered.length > 0) { + lines.push(`Recovered (${report.recovered.length}): ${report.recovered.join(", ")}`); + } + + if (report.cleanedUp.length > 0) { + lines.push(`Cleaned up (${report.cleanedUp.length}): ${report.cleanedUp.join(", ")}`); + } + + if (report.escalated.length > 0) { + lines.push(`Escalated (${report.escalated.length}): ${report.escalated.join(", ")}`); + } + + if (report.skipped.length > 0) { + lines.push(`Skipped (${report.skipped.length}): ${report.skipped.join(", ")}`); + } + + if (report.errors.length > 0) { + lines.push("", "Errors:"); + for (const { sessionId, error } of report.errors) { + lines.push(` ${sessionId}: ${error}`); + } + } + + return lines.join("\n"); +} + +export function createEmptyReport(): RecoveryReport { + return { + timestamp: new Date(), + totalScanned: 0, + recovered: [], + cleanedUp: [], + escalated: [], + skipped: [], + errors: [], + durationMs: 0, + }; +} diff --git a/packages/core/src/recovery/manager.ts b/packages/core/src/recovery/manager.ts new file mode 100644 index 000000000..a204a2f0b --- /dev/null +++ b/packages/core/src/recovery/manager.ts @@ -0,0 +1,190 @@ +import type { OrchestratorConfig, PluginRegistry, Session } from "../types.js"; +import { scanAllSessions, getRecoveryLogPath } from "./scanner.js"; +import { validateSession } from "./validator.js"; +import { executeAction } from "./actions.js"; +import { writeRecoveryLog, createLogEntry, createEmptyReport } from "./logger.js"; +import { + DEFAULT_RECOVERY_CONFIG, + type RecoveryContext, + type RecoveryReport, + type RecoveryResult, + type RecoveryAssessment, + type RecoveryConfig, +} from "./types.js"; + +export interface RecoveryManagerOptions { + config: OrchestratorConfig; + registry: PluginRegistry; + recoveryConfig?: Partial; + dryRun?: boolean; + projectFilter?: string; +} + +export interface RecoveryRunResult { + report: RecoveryReport; + assessments: RecoveryAssessment[]; + results: RecoveryResult[]; + recoveredSessions: Session[]; +} + +export async function runRecovery(options: RecoveryManagerOptions): Promise { + const { config, registry, dryRun = false, projectFilter } = options; + const startTime = Date.now(); + + const recoveryConfig: RecoveryConfig = { + ...DEFAULT_RECOVERY_CONFIG, + ...options.recoveryConfig, + logPath: options.recoveryConfig?.logPath ?? getRecoveryLogPath(config.configPath), + }; + + const context: RecoveryContext = { + configPath: config.configPath, + recoveryConfig, + dryRun, + }; + + const report = createEmptyReport(); + const assessments: RecoveryAssessment[] = []; + const results: RecoveryResult[] = []; + const recoveredSessions: Session[] = []; + + const scannedSessions = scanAllSessions(config, projectFilter); + report.totalScanned = scannedSessions.length; + + for (const scanned of scannedSessions) { + const assessment = await validateSession(scanned, config, registry, recoveryConfig); + assessments.push(assessment); + + if (dryRun) { + const dryRunResult = await executeAction(assessment, config, registry, context); + results.push(dryRunResult); + + switch (dryRunResult.action) { + case "recover": + report.recovered.push(assessment.sessionId); + break; + case "cleanup": + report.cleanedUp.push(assessment.sessionId); + break; + case "escalate": + report.escalated.push(assessment.sessionId); + break; + case "skip": + default: + report.skipped.push(assessment.sessionId); + break; + } + continue; + } + + const result = await executeAction(assessment, config, registry, context); + results.push(result); + + if (result.success) { + switch (result.action) { + case "recover": + report.recovered.push(assessment.sessionId); + if (result.session) { + recoveredSessions.push(result.session); + } + break; + case "cleanup": + report.cleanedUp.push(assessment.sessionId); + break; + case "escalate": + report.escalated.push(assessment.sessionId); + break; + case "skip": + report.skipped.push(assessment.sessionId); + break; + } + } else { + report.errors.push({ + sessionId: assessment.sessionId, + error: result.error || "Unknown error", + }); + } + + const logAction = mapActionToLogAction(result.action, result.success); + writeRecoveryLog( + recoveryConfig.logPath, + createLogEntry(assessment.sessionId, logAction, { + previousStatus: assessment.metadataStatus, + reason: result.reason ?? assessment.reason, + error: result.error, + }), + ); + } + + report.durationMs = Date.now() - startTime; + + return { + report, + assessments, + results, + recoveredSessions, + }; +} + +function mapActionToLogAction( + action: string, + success: boolean, +): "recovered" | "cleaned_up" | "escalated" | "skipped" | "error" { + if (!success) return "error"; + switch (action) { + case "recover": + return "recovered"; + case "cleanup": + return "cleaned_up"; + case "escalate": + return "escalated"; + default: + return "skipped"; + } +} + +export async function recoverSessionById( + sessionId: string, + options: RecoveryManagerOptions, +): Promise { + const { config, registry, dryRun = false } = options; + + const recoveryConfig: RecoveryConfig = { + ...DEFAULT_RECOVERY_CONFIG, + ...options.recoveryConfig, + logPath: options.recoveryConfig?.logPath ?? getRecoveryLogPath(config.configPath), + }; + + const context: RecoveryContext = { + configPath: config.configPath, + recoveryConfig, + dryRun, + }; + + const allSessions = scanAllSessions(config); + const scanned = allSessions.find((s) => s.sessionId === sessionId); + + if (!scanned) { + return null; + } + + const assessment = await validateSession(scanned, config, registry, recoveryConfig); + + const result = await executeAction(assessment, config, registry, context); + + if (dryRun) { + return result; + } + + const logAction = mapActionToLogAction(result.action, result.success); + writeRecoveryLog( + recoveryConfig.logPath, + createLogEntry(sessionId, logAction, { + previousStatus: assessment.metadataStatus, + reason: result.reason ?? assessment.reason, + error: result.error, + }), + ); + + return result; +} diff --git a/packages/core/src/recovery/scanner.ts b/packages/core/src/recovery/scanner.ts new file mode 100644 index 000000000..79a56c56b --- /dev/null +++ b/packages/core/src/recovery/scanner.ts @@ -0,0 +1,48 @@ +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { SessionId, OrchestratorConfig, ProjectConfig } from "../types.js"; +import { listMetadata, readMetadataRaw } from "../metadata.js"; +import { getSessionsDir, generateConfigHash } from "../paths.js"; + +export interface ScannedSession { + sessionId: SessionId; + projectId: string; + project: ProjectConfig; + sessionsDir: string; + rawMetadata: Record; +} + +export function scanAllSessions( + config: OrchestratorConfig, + projectIdFilter?: string, +): ScannedSession[] { + const results: ScannedSession[] = []; + + for (const [projectKey, project] of Object.entries(config.projects)) { + if (projectIdFilter && projectKey !== projectIdFilter) continue; + + const sessionsDir = getSessionsDir(config.configPath, project.path); + if (!existsSync(sessionsDir)) continue; + + for (const file of listMetadata(sessionsDir)) { + const rawMetadata = readMetadataRaw(sessionsDir, file); + if (!rawMetadata) continue; + + results.push({ + sessionId: file, + projectId: projectKey, + project, + sessionsDir, + rawMetadata, + }); + } + } + + return results; +} + +export function getRecoveryLogPath(configPath: string): string { + const hash = generateConfigHash(configPath); + return join(homedir(), ".agent-orchestrator", hash, "recovery.log"); +} diff --git a/packages/core/src/recovery/types.ts b/packages/core/src/recovery/types.ts new file mode 100644 index 000000000..8518a9940 --- /dev/null +++ b/packages/core/src/recovery/types.ts @@ -0,0 +1,216 @@ +/** + * Recovery Types — Session classification and recovery result types. + * + * Part of the orchestrator recovery automation system (Issue #356). + * These types define how sessions are classified and what actions are taken. + */ + +import type { SessionId, SessionStatus, Session, RuntimeHandle, ActivityState } from "../types.js"; + +/** + * Classification of a session's recoverability state. + * + * - live: Session is running normally, just needs to be re-registered + * - dead: Runtime is gone, needs cleanup + * - partial: Some components exist but session is incomplete + * - unrecoverable: Session is in a terminal state (merged, done) - skip + */ +export type RecoveryClassification = "live" | "dead" | "partial" | "unrecoverable"; + +/** + * Action to take for a session during recovery. + * + * - recover: Restore session to in-memory state + * - cleanup: Remove runtime/workspace, archive metadata + * - escalate: Requires manual intervention + * - skip: No action needed + */ +export type RecoveryAction = "recover" | "cleanup" | "escalate" | "skip"; + +/** + * Assessment of a session's state for recovery purposes. + */ +export interface RecoveryAssessment { + /** Session ID being assessed */ + sessionId: SessionId; + + /** Project ID this session belongs to */ + projectId: string; + + /** Overall classification */ + classification: RecoveryClassification; + + /** Recommended action */ + action: RecoveryAction; + + /** Human-readable reason for classification */ + reason: string; + + // --- Runtime state --- + + /** Whether the runtime (tmux/docker) is alive */ + runtimeAlive: boolean; + + /** Runtime handle if available */ + runtimeHandle: RuntimeHandle | null; + + // --- Workspace state --- + + /** Whether the workspace directory exists */ + workspaceExists: boolean; + + /** Workspace path if known */ + workspacePath: string | null; + + // --- Agent state --- + + /** Whether the agent process appears to be running */ + agentProcessRunning: boolean; + + /** Detected agent activity state */ + agentActivity: ActivityState | null; + + // --- Metadata state --- + + /** Whether metadata file is valid/readable */ + metadataValid: boolean; + + /** Current status from metadata */ + metadataStatus: SessionStatus; + + /** Raw metadata key-value pairs */ + rawMetadata: Record; +} + +/** + * Result of attempting to recover a single session. + */ +export interface RecoveryResult { + /** Whether the operation succeeded */ + success: boolean; + + /** Session ID that was processed */ + sessionId: SessionId; + + /** Action that was taken */ + action: RecoveryAction; + + /** Recovered session object (only for 'recover' action) */ + session?: Session; + + /** Whether manual intervention is required */ + requiresManualIntervention?: boolean; + + reason?: string; + + /** Error message if operation failed */ + error?: string; +} + +/** + * Summary report of a recovery operation. + */ +export interface RecoveryReport { + /** When the recovery was run */ + timestamp: Date; + + /** Total sessions scanned */ + totalScanned: number; + + /** Sessions that were recovered (live, restored to memory) */ + recovered: SessionId[]; + + /** Sessions that were cleaned up (dead, resources removed) */ + cleanedUp: SessionId[]; + + /** Sessions that require manual intervention */ + escalated: SessionId[]; + + /** Sessions that were skipped (no action needed) */ + skipped: SessionId[]; + + /** Errors encountered during recovery */ + errors: Array<{ sessionId: SessionId; error: string }>; + + /** Time taken for recovery in milliseconds */ + durationMs: number; +} + +/** + * Entry in the recovery log. + */ +export interface RecoveryLogEntry { + /** ISO timestamp */ + timestamp: string; + + /** Session ID */ + sessionId: SessionId; + + /** Action taken */ + action: "recovered" | "cleaned_up" | "escalated" | "skipped" | "error"; + + /** Previous status (for recovered sessions) */ + previousStatus?: SessionStatus; + + /** Reason for action (for cleanup/escalate) */ + reason?: string; + + /** Error message (for error action) */ + error?: string; + + /** Additional details */ + details?: Record; +} + +/** + * Configuration for recovery behavior. + */ +export interface RecoveryConfig { + /** Enable automatic recovery on orchestrator startup */ + enabled: boolean; + + /** Maximum time for recovery phase in milliseconds */ + timeoutMs: number; + + /** Number of concurrent validation tasks */ + parallelValidation: number; + + /** Path to recovery log file */ + logPath: string; + + /** Automatically cleanup dead sessions */ + autoCleanup: boolean; + + /** Escalate partial sessions (vs. auto-cleanup) */ + escalatePartial: boolean; + + /** Maximum recovery attempts before escalating */ + maxRecoveryAttempts: number; +} + +/** + * Default recovery configuration. + */ +export const DEFAULT_RECOVERY_CONFIG: RecoveryConfig = { + enabled: true, + timeoutMs: 30_000, + parallelValidation: 5, + logPath: "", // Will be set to ~/.agent-orchestrator/recovery.log + autoCleanup: true, + escalatePartial: true, + maxRecoveryAttempts: 3, +}; + +/** + * Context passed to recovery functions. + */ +export interface RecoveryContext { + /** Root config path for the orchestrator */ + configPath: string; + + /** Recovery configuration */ + recoveryConfig: RecoveryConfig; + + /** Whether this is a dry run (no actual changes) */ + dryRun: boolean; +} diff --git a/packages/core/src/recovery/validator.ts b/packages/core/src/recovery/validator.ts new file mode 100644 index 000000000..8b4534991 --- /dev/null +++ b/packages/core/src/recovery/validator.ts @@ -0,0 +1,181 @@ +import { existsSync } from "node:fs"; +import { + TERMINAL_STATUSES as TERMINAL_STATUSES_SET, + type OrchestratorConfig, + type PluginRegistry, + type Runtime, + type Agent, + type Workspace, + type RuntimeHandle, + type SessionStatus, + type ActivityState, +} from "../types.js"; +import { safeJsonParse, validateStatus } from "../utils/validation.js"; +import type { ScannedSession } from "./scanner.js"; +import { + DEFAULT_RECOVERY_CONFIG, + type RecoveryAssessment, + type RecoveryClassification, + type RecoveryAction, + type RecoveryConfig, +} from "./types.js"; + +export async function validateSession( + scanned: ScannedSession, + config: OrchestratorConfig, + registry: PluginRegistry, + recoveryConfigInput?: Partial, +): Promise { + const { sessionId, projectId, project, rawMetadata } = scanned; + + const runtimeName = project.runtime ?? config.defaults.runtime; + const agentName = rawMetadata["agent"] ?? project.agent ?? config.defaults.agent; + const workspaceName = project.workspace ?? config.defaults.workspace; + + const runtime = registry.get("runtime", runtimeName); + const agent = registry.get("agent", agentName); + const workspace = registry.get("workspace", workspaceName); + + const workspacePath = rawMetadata["worktree"] || null; + const runtimeHandleStr = rawMetadata["runtimeHandle"]; + const runtimeHandle = runtimeHandleStr ? safeJsonParse(runtimeHandleStr) : null; + const metadataStatus = validateStatus(rawMetadata["status"]); + const recoveryConfig: RecoveryConfig = { + ...DEFAULT_RECOVERY_CONFIG, + ...(recoveryConfigInput ?? {}), + }; + + let runtimeAlive = false; + if (runtime && runtimeHandle) { + try { + runtimeAlive = await runtime.isAlive(runtimeHandle); + } catch { + runtimeAlive = false; + } + } + + let workspaceExists = false; + if (workspacePath) { + try { + workspaceExists = existsSync(workspacePath); + } catch { + workspaceExists = false; + } + if (!workspaceExists && workspace?.exists) { + try { + workspaceExists = await workspace.exists(workspacePath); + } catch { + workspaceExists = false; + } + } + } + + let agentProcessRunning = false; + const agentActivity: ActivityState | null = null; + if (agent && runtimeHandle) { + try { + agentProcessRunning = await agent.isProcessRunning(runtimeHandle); + } catch { + agentProcessRunning = false; + } + } + + const metadataValid = Object.keys(rawMetadata).length > 0; + const classification = classifySession( + runtimeAlive, + workspaceExists, + agentProcessRunning, + metadataStatus, + ); + const action = determineAction(classification, metadataStatus, recoveryConfig); + + return { + sessionId, + projectId, + classification, + action, + reason: getReason(classification, runtimeAlive, workspaceExists, agentProcessRunning), + runtimeAlive, + runtimeHandle, + workspaceExists, + workspacePath, + agentProcessRunning, + agentActivity, + metadataValid, + metadataStatus, + rawMetadata, + }; +} + +function classifySession( + runtimeAlive: boolean, + workspaceExists: boolean, + agentProcessRunning: boolean, + metadataStatus: SessionStatus, +): RecoveryClassification { + if (runtimeAlive && workspaceExists && agentProcessRunning) { + return "live"; + } + + if (!runtimeAlive && !workspaceExists) { + if (TERMINAL_STATUSES_SET.has(metadataStatus)) { + return "unrecoverable"; + } + return "dead"; + } + + if (runtimeAlive && !workspaceExists) { + return "partial"; + } + + if (!runtimeAlive && workspaceExists) { + return "dead"; + } + + if (runtimeAlive && workspaceExists && !agentProcessRunning) { + return "partial"; + } + + return "partial"; +} + +function determineAction( + classification: RecoveryClassification, + _metadataStatus: SessionStatus, + recoveryConfig: RecoveryConfig, +): RecoveryAction { + switch (classification) { + case "live": + return "recover"; + case "dead": + return recoveryConfig.autoCleanup ? "cleanup" : "escalate"; + case "partial": + return recoveryConfig.escalatePartial ? "escalate" : "cleanup"; + case "unrecoverable": + return "skip"; + default: + return "skip"; + } +} + +function getReason( + classification: RecoveryClassification, + runtimeAlive: boolean, + workspaceExists: boolean, + agentProcessRunning: boolean, +): string { + switch (classification) { + case "live": + return "Session is running normally"; + case "dead": + return `Runtime ${runtimeAlive ? "alive" : "dead"}, workspace ${workspaceExists ? "exists" : "missing"}`; + case "partial": + return `Incomplete state: runtime=${runtimeAlive}, workspace=${workspaceExists}, agent=${agentProcessRunning}`; + case "unrecoverable": + return "Session is in terminal state"; + default: + return "Unknown classification"; + } +} + +export { classifySession, determineAction }; diff --git a/packages/core/src/session-manager.ts b/packages/core/src/session-manager.ts index 3a072f9cb..4b5242330 100644 --- a/packages/core/src/session-manager.ts +++ b/packages/core/src/session-manager.ts @@ -28,7 +28,6 @@ import { type SessionId, type SessionSpawnConfig, type OrchestratorSpawnConfig, - type SessionStatus, type CleanupResult, type ClaimPROptions, type ClaimPRResult, @@ -71,6 +70,8 @@ import { GLOBAL_PAUSE_SOURCE_KEY, parsePauseUntil, } from "./global-pause.js"; +import { sessionFromMetadata } from "./utils/session-from-metadata.js"; +import { safeJsonParse } from "./utils/validation.js"; const execFileAsync = promisify(execFile); const OPENCODE_DISCOVERY_TIMEOUT_MS = 2_000; @@ -191,35 +192,6 @@ function getNextSessionNumber(existingSessions: string[], prefix: string): numbe return max + 1; } -/** Safely parse JSON, returning null on failure. */ -function safeJsonParse(str: string): T | null { - try { - return JSON.parse(str) as T; - } catch { - return null; - } -} - -/** Valid session statuses for validation. */ -const VALID_STATUSES: ReadonlySet = new Set([ - "spawning", - "working", - "pr_open", - "ci_failed", - "review_pending", - "changes_requested", - "approved", - "mergeable", - "merged", - "cleanup", - "needs_input", - "stuck", - "errored", - "killed", - "done", - "terminated", -]); - const PR_TRACKING_STATUSES: ReadonlySet = new Set([ "pr_open", "ci_failed", @@ -239,14 +211,6 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -/** Validate and normalize a status string. */ -function validateStatus(raw: string | undefined): SessionStatus { - // Bash scripts write "starting" — treat as "working" - if (raw === "starting") return "working"; - if (raw && VALID_STATUSES.has(raw)) return raw as SessionStatus; - return "spawning"; -} - /** Reconstruct a Session object from raw metadata key=value pairs. */ function metadataToSession( sessionId: SessionId, @@ -254,42 +218,10 @@ function metadataToSession( createdAt?: Date, modifiedAt?: Date, ): Session { - return { - id: sessionId, - projectId: meta["project"] ?? "", - status: validateStatus(meta["status"]), - activity: null, - branch: meta["branch"] || null, - issueId: meta["issue"] || null, - pr: meta["pr"] - ? (() => { - // Parse owner/repo from GitHub PR URL: https://github.com/owner/repo/pull/123 - const prUrl = meta["pr"]; - const ghMatch = prUrl.match(/github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/); - return { - number: ghMatch - ? parseInt(ghMatch[3], 10) - : parseInt(prUrl.match(/\/(\d+)$/)?.[1] ?? "0", 10), - url: prUrl, - title: "", - owner: ghMatch?.[1] ?? "", - repo: ghMatch?.[2] ?? "", - branch: meta["branch"] ?? "", - baseBranch: "", - isDraft: false, - }; - })() - : null, - workspacePath: meta["worktree"] || null, - runtimeHandle: meta["runtimeHandle"] - ? safeJsonParse(meta["runtimeHandle"]) - : null, - agentInfo: meta["summary"] ? { summary: meta["summary"], agentSessionId: null } : null, - createdAt: meta["createdAt"] ? new Date(meta["createdAt"]) : (createdAt ?? new Date()), + return sessionFromMetadata(sessionId, meta, { + createdAt, lastActivityAt: modifiedAt ?? new Date(), - restoredAt: meta["restoredAt"] ? new Date(meta["restoredAt"]) : undefined, - metadata: meta, - }; + }); } export interface SessionManagerDeps { diff --git a/packages/core/src/utils/pr.ts b/packages/core/src/utils/pr.ts new file mode 100644 index 000000000..0f4fd977d --- /dev/null +++ b/packages/core/src/utils/pr.ts @@ -0,0 +1,31 @@ +import type { PRInfo } from "../types.js"; + +export type ParsedPrUrl = Pick; + +const GITHUB_PR_URL_REGEX = /github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/; +const TRAILING_NUMBER_REGEX = /\/(\d+)$/; + +export function parsePrFromUrl(prUrl: string): ParsedPrUrl | null { + const githubMatch = prUrl.match(GITHUB_PR_URL_REGEX); + if (githubMatch) { + const [, owner, repo, prNumber] = githubMatch; + return { + owner, + repo, + number: parseInt(prNumber, 10), + url: prUrl, + }; + } + + const trailingNumberMatch = prUrl.match(TRAILING_NUMBER_REGEX); + if (trailingNumberMatch) { + return { + owner: "", + repo: "", + number: parseInt(trailingNumberMatch[1], 10), + url: prUrl, + }; + } + + return null; +} diff --git a/packages/core/src/utils/session-from-metadata.ts b/packages/core/src/utils/session-from-metadata.ts new file mode 100644 index 000000000..50996fda8 --- /dev/null +++ b/packages/core/src/utils/session-from-metadata.ts @@ -0,0 +1,55 @@ +import type { RuntimeHandle, Session, SessionId, SessionStatus } from "../types.js"; +import { parsePrFromUrl } from "./pr.js"; +import { safeJsonParse, validateStatus } from "./validation.js"; + +interface SessionFromMetadataOptions { + status?: SessionStatus; + activity?: Session["activity"]; + runtimeHandle?: RuntimeHandle | null; + createdAt?: Date; + lastActivityAt?: Date; + restoredAt?: Date; +} + +export function sessionFromMetadata( + sessionId: SessionId, + meta: Record, + options: SessionFromMetadataOptions = {}, +): Session { + return { + id: sessionId, + projectId: meta["project"] ?? "", + status: options.status ?? validateStatus(meta["status"]), + activity: options.activity ?? null, + branch: meta["branch"] || null, + issueId: meta["issue"] || null, + pr: meta["pr"] + ? (() => { + const parsed = parsePrFromUrl(meta["pr"]); + return { + number: parsed?.number ?? 0, + url: meta["pr"], + title: "", + owner: parsed?.owner ?? "", + repo: parsed?.repo ?? "", + branch: meta["branch"] ?? "", + baseBranch: "", + isDraft: false, + }; + })() + : null, + workspacePath: meta["worktree"] || null, + runtimeHandle: + options.runtimeHandle !== undefined + ? options.runtimeHandle + : meta["runtimeHandle"] + ? safeJsonParse(meta["runtimeHandle"]) + : null, + agentInfo: meta["summary"] ? { summary: meta["summary"], agentSessionId: null } : null, + createdAt: meta["createdAt"] ? new Date(meta["createdAt"]) : (options.createdAt ?? new Date()), + lastActivityAt: options.lastActivityAt ?? new Date(), + restoredAt: + options.restoredAt ?? (meta["restoredAt"] ? new Date(meta["restoredAt"]) : undefined), + metadata: meta, + }; +} diff --git a/packages/core/src/utils/validation.ts b/packages/core/src/utils/validation.ts new file mode 100644 index 000000000..0e9dd744f --- /dev/null +++ b/packages/core/src/utils/validation.ts @@ -0,0 +1,42 @@ +/** + * Shared validation utilities. + */ + +import type { SessionStatus } from "../types.js"; + +/** Valid session statuses for validation. */ +const VALID_STATUSES: ReadonlySet = new Set([ + "spawning", + "working", + "pr_open", + "ci_failed", + "review_pending", + "changes_requested", + "approved", + "mergeable", + "merged", + "cleanup", + "needs_input", + "stuck", + "errored", + "killed", + "done", + "terminated", +]); + +/** Safely parse JSON, returning null on failure. */ +export function safeJsonParse(str: string): T | null { + try { + return JSON.parse(str) as T; + } catch { + return null; + } +} + +/** Validate and normalize a status string. */ +export function validateStatus(raw: string | undefined): SessionStatus { + // Bash scripts write "starting" — treat as "working" + if (raw === "starting") return "working"; + if (raw && VALID_STATUSES.has(raw)) return raw as SessionStatus; + return "spawning"; +}