From 9fa0a65ec93e719f03e82f79d82d940ea3b2ce6d Mon Sep 17 00:00:00 2001 From: Harsh Batheja <40922251+harsh-batheja@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:43:42 +0530 Subject: [PATCH] feat: add orchestrator recovery automation (#356) (#362) * fix: preserve PR number in recovery and dedupe validation utilities Bugbot #2908822306: Use parsePrFromUrl utility to correctly extract PR number from URL instead of defaulting to 0. Applied to both recovery/actions.ts and session-manager.ts for consistency. Bugbot #2908822310: Extract safeJsonParse and validateStatus to shared utils/validation.ts to eliminate duplication between recovery/validator.ts and session-manager.ts. Clean rebuild from origin/main with only recovery-specific changes. * fix: add PR number fallback and remove unused import Bugbot #2908822306: Add fallback regex to extract PR number from URL ending when full GitHub URL pattern doesn't match (e.g., non-GitHub URLs). Bugbot #2908822310: Remove unused SessionStatus import from session-manager.ts (now imported from utils/validation.ts). Tests: 406 passed, typecheck clean. * fix(core): harden recovery action selection and escalation * fix(core): preserve recovered agent summary metadata * fix(core): reuse canonical PR type in URL parser Avoid shadowing the core PRInfo type in the recovery URL parser while keeping the non-GitHub trailing-number fallback covered by tests. * fix(core): align recovery metadata and session reconstruction Persist restored timestamps under the canonical metadata key, share session reconstruction logic between recovery and session loading, and log the real escalation reason when recovery aborts on retry limits. * fix(core): keep dry-run escalation reasons accurate Use the assessment's actual escalation reason in dry-run results so preview output matches real execution behavior for partial-session escalations. * fix(core): dedupe recovery scanning and honor custom log path Reuse metadata listing rules in scanner to avoid duplicated session ID filters, and preserve user-supplied recovery logPath in recovery manager APIs. * fix(core): align dry-run recovery behavior with real actions Compute recovery escalation decisions before dry-run returns and route single-session dry runs through executeAction so action-specific fields (reason/manual-intervention) are preserved. * fix(core): derive dry-run recovery report from action execution Run dry-run recovery classification through executeAction so report actions match real execution logic, including max-attempt escalation decisions. --- .../src/__tests__/recovery-actions.test.ts | 284 ++++++++++++++++++ packages/core/src/__tests__/utils.test.ts | 25 ++ packages/core/src/recovery/actions.ts | 229 ++++++++++++++ packages/core/src/recovery/index.ts | 32 ++ packages/core/src/recovery/logger.ts | 78 +++++ packages/core/src/recovery/manager.ts | 190 ++++++++++++ packages/core/src/recovery/scanner.ts | 48 +++ packages/core/src/recovery/types.ts | 216 +++++++++++++ packages/core/src/recovery/validator.ts | 181 +++++++++++ packages/core/src/session-manager.ts | 78 +---- packages/core/src/utils/pr.ts | 31 ++ .../core/src/utils/session-from-metadata.ts | 55 ++++ packages/core/src/utils/validation.ts | 42 +++ 13 files changed, 1416 insertions(+), 73 deletions(-) create mode 100644 packages/core/src/__tests__/recovery-actions.test.ts create mode 100644 packages/core/src/recovery/actions.ts create mode 100644 packages/core/src/recovery/index.ts create mode 100644 packages/core/src/recovery/logger.ts create mode 100644 packages/core/src/recovery/manager.ts create mode 100644 packages/core/src/recovery/scanner.ts create mode 100644 packages/core/src/recovery/types.ts create mode 100644 packages/core/src/recovery/validator.ts create mode 100644 packages/core/src/utils/pr.ts create mode 100644 packages/core/src/utils/session-from-metadata.ts create mode 100644 packages/core/src/utils/validation.ts diff --git a/packages/core/src/__tests__/recovery-actions.test.ts b/packages/core/src/__tests__/recovery-actions.test.ts new file mode 100644 index 000000000..1f9cfc392 --- /dev/null +++ b/packages/core/src/__tests__/recovery-actions.test.ts @@ -0,0 +1,284 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { readMetadataRaw } from "../metadata.js"; +import { getSessionsDir } from "../paths.js"; +import { escalateSession, recoverSession } from "../recovery/actions.js"; +import { runRecovery } from "../recovery/manager.js"; +import { getRecoveryLogPath, scanAllSessions } from "../recovery/scanner.js"; +import { + DEFAULT_RECOVERY_CONFIG, + type RecoveryAssessment, + type RecoveryContext, +} from "../recovery/types.js"; +import type { OrchestratorConfig, PluginRegistry } from "../types.js"; + +function makeConfig(rootDir: string): OrchestratorConfig { + return { + configPath: join(rootDir, "agent-orchestrator.yaml"), + port: 3000, + readyThresholdMs: 300_000, + defaults: { + runtime: "tmux", + agent: "claude-code", + workspace: "worktree", + notifiers: ["desktop"], + }, + projects: { + app: { + name: "app", + repo: "org/repo", + path: join(rootDir, "project"), + defaultBranch: "main", + sessionPrefix: "app", + }, + }, + notifiers: {}, + notificationRouting: { + urgent: ["desktop"], + action: ["desktop"], + warning: ["desktop"], + info: ["desktop"], + }, + reactions: {}, + }; +} + +function makeRegistry(): PluginRegistry { + return { + register: vi.fn(), + get: vi.fn().mockReturnValue(null), + list: vi.fn().mockReturnValue([]), + loadBuiltins: vi.fn().mockResolvedValue(undefined), + loadFromConfig: vi.fn().mockResolvedValue(undefined), + }; +} + +function makeAssessment(overrides: Partial = {}): RecoveryAssessment { + return { + sessionId: "app-1", + projectId: "app", + classification: "live", + action: "recover", + reason: "Session is running normally", + runtimeAlive: true, + runtimeHandle: { id: "rt-1", runtimeName: "tmux", data: {} }, + workspaceExists: true, + workspacePath: "/tmp/worktree", + agentProcessRunning: true, + agentActivity: "active", + metadataValid: true, + metadataStatus: "working", + rawMetadata: { + project: "app", + branch: "feat/test", + issue: "123", + pr: "https://github.com/org/repo/pull/42", + createdAt: "2025-01-01T00:00:00.000Z", + status: "working", + summary: "Recovered summary", + }, + ...overrides, + }; +} + +function makeContext(rootDir: string, overrides: Partial = {}): RecoveryContext { + return { + configPath: join(rootDir, "agent-orchestrator.yaml"), + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: join(rootDir, "recovery.log"), + }, + dryRun: false, + ...overrides, + }; +} + +describe("recoverSession", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("persists restoredAt and returns a session with restoredAt", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment(); + const context = makeContext(rootDir); + + const result = await recoverSession(assessment, config, registry, context); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + const metadata = readMetadataRaw(sessionsDir, assessment.sessionId); + + expect(result.success).toBe(true); + expect(result.session?.restoredAt).toBeInstanceOf(Date); + expect(metadata?.["restoredAt"]).toBeDefined(); + expect(metadata?.["recoveredAt"]).toBeUndefined(); + }); + + it("returns the max-attempt reason when recovery escalates", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment({ + rawMetadata: { + ...makeAssessment().rawMetadata, + recoveryCount: "3", + }, + }); + const context = makeContext(rootDir, { + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: join(rootDir, "recovery.log"), + maxRecoveryAttempts: 3, + }, + }); + + const result = await recoverSession(assessment, config, registry, context); + + expect(result.success).toBe(true); + expect(result.action).toBe("escalate"); + expect(result.reason).toBe("Exceeded max recovery attempts (3)"); + }); + + it("dry-run recovery reports escalate when attempts exceed limit", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment({ + rawMetadata: { + ...makeAssessment().rawMetadata, + recoveryCount: "3", + }, + }); + const context = makeContext(rootDir, { + dryRun: true, + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: join(rootDir, "recovery.log"), + maxRecoveryAttempts: 3, + }, + }); + + const result = await recoverSession(assessment, config, registry, context); + + expect(result.success).toBe(true); + expect(result.action).toBe("escalate"); + expect(result.requiresManualIntervention).toBe(true); + expect(result.reason).toBe("Exceeded max recovery attempts (3)"); + }); +}); + +describe("escalateSession", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("uses the assessment reason during dry runs", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const registry = makeRegistry(); + const assessment = makeAssessment({ + action: "escalate", + classification: "partial", + reason: "Workspace exists but runtime is missing", + }); + const context = makeContext(rootDir, { dryRun: true }); + + const result = await escalateSession(assessment, config, registry, context); + + expect(result.success).toBe(true); + expect(result.action).toBe("escalate"); + expect(result.reason).toBe("Workspace exists but runtime is missing"); + }); +}); + +describe("recovery manager and scanner", () => { + let rootDir: string; + + afterEach(() => { + if (rootDir) { + rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("respects custom recovery logPath in manager options", async () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + mkdirSync(sessionsDir, { recursive: true }); + writeFileSync( + join(sessionsDir, "app-1"), + "project=app\nstatus=terminated\nworktree=/tmp/worktree\n", + "utf-8", + ); + + const customLogPath = join(rootDir, "custom-recovery.log"); + const registry = makeRegistry(); + + await runRecovery({ + config, + registry, + recoveryConfig: { + ...DEFAULT_RECOVERY_CONFIG, + logPath: customLogPath, + }, + }); + + expect(existsSync(customLogPath)).toBe(true); + expect(readFileSync(customLogPath, "utf-8")).toContain('"sessionId":"app-1"'); + + const defaultLogPath = getRecoveryLogPath(config.configPath); + expect(defaultLogPath).not.toBe(customLogPath); + }); + + it("scans sessions using metadata listing rules", () => { + rootDir = join(tmpdir(), `ao-recovery-${randomUUID()}`); + mkdirSync(rootDir, { recursive: true }); + mkdirSync(join(rootDir, "project"), { recursive: true }); + writeFileSync(join(rootDir, "agent-orchestrator.yaml"), "projects: {}\n", "utf-8"); + + const config = makeConfig(rootDir); + const sessionsDir = getSessionsDir(config.configPath, config.projects.app.path); + mkdirSync(sessionsDir, { recursive: true }); + + writeFileSync(join(sessionsDir, "app-1"), "project=app\nstatus=working\n", "utf-8"); + writeFileSync(join(sessionsDir, ".tmp"), "project=app\n", "utf-8"); + writeFileSync(join(sessionsDir, "bad.session"), "project=app\n", "utf-8"); + + const scanned = scanAllSessions(config); + + expect(scanned).toHaveLength(1); + expect(scanned[0]?.sessionId).toBe("app-1"); + }); +}); diff --git a/packages/core/src/__tests__/utils.test.ts b/packages/core/src/__tests__/utils.test.ts index 2c22d6007..341040f39 100644 --- a/packages/core/src/__tests__/utils.test.ts +++ b/packages/core/src/__tests__/utils.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { isRetryableHttpStatus, normalizeRetryConfig, readLastJsonlEntry } from "../utils.js"; +import { parsePrFromUrl } from "../utils/pr.js"; describe("readLastJsonlEntry", () => { let tmpDir: string; @@ -114,3 +115,27 @@ describe("retry utilities", () => { }); }); }); + +describe("parsePrFromUrl", () => { + it("parses GitHub PR URLs", () => { + expect(parsePrFromUrl("https://github.com/foo/bar/pull/123")).toEqual({ + owner: "foo", + repo: "bar", + number: 123, + url: "https://github.com/foo/bar/pull/123", + }); + }); + + it("falls back to trailing number for non-GitHub URLs", () => { + expect(parsePrFromUrl("https://gitlab.com/foo/bar/-/merge_requests/456")).toEqual({ + owner: "", + repo: "", + number: 456, + url: "https://gitlab.com/foo/bar/-/merge_requests/456", + }); + }); + + it("returns null when the URL has no PR number", () => { + expect(parsePrFromUrl("https://example.com/foo/bar/pull/not-a-number")).toBeNull(); + }); +}); diff --git a/packages/core/src/recovery/actions.ts b/packages/core/src/recovery/actions.ts new file mode 100644 index 000000000..11d4bb762 --- /dev/null +++ b/packages/core/src/recovery/actions.ts @@ -0,0 +1,229 @@ +import type { OrchestratorConfig, PluginRegistry, Runtime, Workspace } from "../types.js"; +import { updateMetadata, deleteMetadata } from "../metadata.js"; +import { getSessionsDir } from "../paths.js"; +import { validateStatus } from "../utils/validation.js"; +import { sessionFromMetadata } from "../utils/session-from-metadata.js"; +import type { RecoveryAssessment, RecoveryResult, RecoveryContext } from "./types.js"; + +export async function recoverSession( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + registry: PluginRegistry, + context: RecoveryContext, +): Promise { + const { sessionId, projectId, rawMetadata } = assessment; + const recoveryCount = rawMetadata["recoveryCount"] + ? parseInt(rawMetadata["recoveryCount"], 10) + 1 + : 1; + + if (context.dryRun) { + if (recoveryCount > context.recoveryConfig.maxRecoveryAttempts) { + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`, + }; + } + + return { + success: true, + sessionId, + action: "recover", + }; + } + + try { + const now = new Date().toISOString(); + const preservedStatus = validateStatus(rawMetadata["status"]); + + const project = config.projects[projectId]; + const sessionsDir = getSessionsDir(config.configPath, project.path); + + if (recoveryCount > context.recoveryConfig.maxRecoveryAttempts) { + updateMetadata(sessionsDir, sessionId, { + status: "stuck", + escalatedAt: now, + escalationReason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`, + recoveryCount: String(recoveryCount), + }); + + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason: `Exceeded max recovery attempts (${context.recoveryConfig.maxRecoveryAttempts})`, + }; + } + + updateMetadata(sessionsDir, sessionId, { + status: preservedStatus, + restoredAt: now, + recoveryCount: String(recoveryCount), + }); + + const updatedMetadata = { + ...rawMetadata, + status: preservedStatus, + restoredAt: now, + recoveryCount: String(recoveryCount), + }; + + const session = sessionFromMetadata(sessionId, updatedMetadata, { + status: preservedStatus, + runtimeHandle: assessment.runtimeHandle, + lastActivityAt: new Date(), + restoredAt: new Date(now), + }); + + return { + success: true, + sessionId, + action: "recover", + session, + }; + } catch (error) { + return { + success: false, + sessionId, + action: "recover", + error: error instanceof Error ? error.message : String(error), + }; + } +} + +export async function cleanupSession( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + registry: PluginRegistry, + context: RecoveryContext, +): Promise { + const { sessionId, projectId, rawMetadata, runtimeAlive, workspaceExists } = assessment; + + if (context.dryRun) { + return { + success: true, + sessionId, + action: "cleanup", + }; + } + + try { + const project = config.projects[projectId]; + const runtimeName = project.runtime ?? config.defaults.runtime; + const workspaceName = project.workspace ?? config.defaults.workspace; + const runtime = registry.get("runtime", runtimeName); + const workspace = registry.get("workspace", workspaceName); + + if (runtimeAlive && assessment.runtimeHandle && runtime) { + try { + await runtime.destroy(assessment.runtimeHandle); + } catch { + // ignore cleanup errors + } + } + + const workspacePath = rawMetadata["worktree"]; + if (workspacePath && workspaceExists && workspace) { + try { + await workspace.destroy(workspacePath); + } catch { + // ignore cleanup errors + } + } + + const sessionsDir = getSessionsDir(config.configPath, project.path); + + updateMetadata(sessionsDir, sessionId, { + status: "terminated", + terminatedAt: new Date().toISOString(), + terminationReason: "cleanup", + }); + + deleteMetadata(sessionsDir, sessionId, true); + + return { + success: true, + sessionId, + action: "cleanup", + }; + } catch (error) { + return { + success: false, + sessionId, + action: "cleanup", + error: error instanceof Error ? error.message : String(error), + }; + } +} + +export async function escalateSession( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + _registry: PluginRegistry, + context: RecoveryContext, +): Promise { + const { sessionId, projectId, reason } = assessment; + + if (context.dryRun) { + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason, + }; + } + + try { + const project = config.projects[projectId]; + const sessionsDir = getSessionsDir(config.configPath, project.path); + + updateMetadata(sessionsDir, sessionId, { + status: "stuck", + escalatedAt: new Date().toISOString(), + escalationReason: reason, + }); + + return { + success: true, + sessionId, + action: "escalate", + requiresManualIntervention: true, + reason, + }; + } catch (error) { + return { + success: false, + sessionId, + action: "escalate", + error: error instanceof Error ? error.message : String(error), + requiresManualIntervention: true, + }; + } +} + +export async function executeAction( + assessment: RecoveryAssessment, + config: OrchestratorConfig, + registry: PluginRegistry, + context: RecoveryContext, +): Promise { + switch (assessment.action) { + case "recover": + return recoverSession(assessment, config, registry, context); + case "cleanup": + return cleanupSession(assessment, config, registry, context); + case "escalate": + return escalateSession(assessment, config, registry, context); + case "skip": + default: + return { + success: true, + sessionId: assessment.sessionId, + action: "skip", + }; + } +} diff --git a/packages/core/src/recovery/index.ts b/packages/core/src/recovery/index.ts new file mode 100644 index 000000000..40dd5933f --- /dev/null +++ b/packages/core/src/recovery/index.ts @@ -0,0 +1,32 @@ +export type { + RecoveryClassification, + RecoveryAction, + RecoveryAssessment, + RecoveryResult, + RecoveryReport, + RecoveryLogEntry, + RecoveryConfig, + RecoveryContext, +} from "./types.js"; + +export { DEFAULT_RECOVERY_CONFIG } from "./types.js"; + +export { scanAllSessions, getRecoveryLogPath, type ScannedSession } from "./scanner.js"; + +export { validateSession, classifySession, determineAction } from "./validator.js"; + +export { recoverSession, cleanupSession, escalateSession, executeAction } from "./actions.js"; + +export { + writeRecoveryLog, + createLogEntry, + formatRecoveryReport, + createEmptyReport, +} from "./logger.js"; + +export { + runRecovery, + recoverSessionById, + type RecoveryManagerOptions, + type RecoveryRunResult, +} from "./manager.js"; diff --git a/packages/core/src/recovery/logger.ts b/packages/core/src/recovery/logger.ts new file mode 100644 index 000000000..0f9bfc4db --- /dev/null +++ b/packages/core/src/recovery/logger.ts @@ -0,0 +1,78 @@ +import { appendFileSync, existsSync, mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import type { SessionId, SessionStatus } from "../types.js"; +import type { RecoveryLogEntry, RecoveryReport } from "./types.js"; + +export function writeRecoveryLog(logPath: string, entry: RecoveryLogEntry): void { + const dir = dirname(logPath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + const line = JSON.stringify(entry) + "\n"; + appendFileSync(logPath, line, "utf-8"); +} + +export function createLogEntry( + sessionId: SessionId, + action: RecoveryLogEntry["action"], + options?: { + previousStatus?: SessionStatus; + reason?: string; + error?: string; + details?: Record; + }, +): RecoveryLogEntry { + return { + timestamp: new Date().toISOString(), + sessionId, + action, + ...options, + }; +} + +export function formatRecoveryReport(report: RecoveryReport): string { + const lines: string[] = [ + `Recovery Report - ${report.timestamp.toISOString()}`, + `Duration: ${report.durationMs}ms`, + `Sessions scanned: ${report.totalScanned}`, + "", + ]; + + if (report.recovered.length > 0) { + lines.push(`Recovered (${report.recovered.length}): ${report.recovered.join(", ")}`); + } + + if (report.cleanedUp.length > 0) { + lines.push(`Cleaned up (${report.cleanedUp.length}): ${report.cleanedUp.join(", ")}`); + } + + if (report.escalated.length > 0) { + lines.push(`Escalated (${report.escalated.length}): ${report.escalated.join(", ")}`); + } + + if (report.skipped.length > 0) { + lines.push(`Skipped (${report.skipped.length}): ${report.skipped.join(", ")}`); + } + + if (report.errors.length > 0) { + lines.push("", "Errors:"); + for (const { sessionId, error } of report.errors) { + lines.push(` ${sessionId}: ${error}`); + } + } + + return lines.join("\n"); +} + +export function createEmptyReport(): RecoveryReport { + return { + timestamp: new Date(), + totalScanned: 0, + recovered: [], + cleanedUp: [], + escalated: [], + skipped: [], + errors: [], + durationMs: 0, + }; +} diff --git a/packages/core/src/recovery/manager.ts b/packages/core/src/recovery/manager.ts new file mode 100644 index 000000000..a204a2f0b --- /dev/null +++ b/packages/core/src/recovery/manager.ts @@ -0,0 +1,190 @@ +import type { OrchestratorConfig, PluginRegistry, Session } from "../types.js"; +import { scanAllSessions, getRecoveryLogPath } from "./scanner.js"; +import { validateSession } from "./validator.js"; +import { executeAction } from "./actions.js"; +import { writeRecoveryLog, createLogEntry, createEmptyReport } from "./logger.js"; +import { + DEFAULT_RECOVERY_CONFIG, + type RecoveryContext, + type RecoveryReport, + type RecoveryResult, + type RecoveryAssessment, + type RecoveryConfig, +} from "./types.js"; + +export interface RecoveryManagerOptions { + config: OrchestratorConfig; + registry: PluginRegistry; + recoveryConfig?: Partial; + dryRun?: boolean; + projectFilter?: string; +} + +export interface RecoveryRunResult { + report: RecoveryReport; + assessments: RecoveryAssessment[]; + results: RecoveryResult[]; + recoveredSessions: Session[]; +} + +export async function runRecovery(options: RecoveryManagerOptions): Promise { + const { config, registry, dryRun = false, projectFilter } = options; + const startTime = Date.now(); + + const recoveryConfig: RecoveryConfig = { + ...DEFAULT_RECOVERY_CONFIG, + ...options.recoveryConfig, + logPath: options.recoveryConfig?.logPath ?? getRecoveryLogPath(config.configPath), + }; + + const context: RecoveryContext = { + configPath: config.configPath, + recoveryConfig, + dryRun, + }; + + const report = createEmptyReport(); + const assessments: RecoveryAssessment[] = []; + const results: RecoveryResult[] = []; + const recoveredSessions: Session[] = []; + + const scannedSessions = scanAllSessions(config, projectFilter); + report.totalScanned = scannedSessions.length; + + for (const scanned of scannedSessions) { + const assessment = await validateSession(scanned, config, registry, recoveryConfig); + assessments.push(assessment); + + if (dryRun) { + const dryRunResult = await executeAction(assessment, config, registry, context); + results.push(dryRunResult); + + switch (dryRunResult.action) { + case "recover": + report.recovered.push(assessment.sessionId); + break; + case "cleanup": + report.cleanedUp.push(assessment.sessionId); + break; + case "escalate": + report.escalated.push(assessment.sessionId); + break; + case "skip": + default: + report.skipped.push(assessment.sessionId); + break; + } + continue; + } + + const result = await executeAction(assessment, config, registry, context); + results.push(result); + + if (result.success) { + switch (result.action) { + case "recover": + report.recovered.push(assessment.sessionId); + if (result.session) { + recoveredSessions.push(result.session); + } + break; + case "cleanup": + report.cleanedUp.push(assessment.sessionId); + break; + case "escalate": + report.escalated.push(assessment.sessionId); + break; + case "skip": + report.skipped.push(assessment.sessionId); + break; + } + } else { + report.errors.push({ + sessionId: assessment.sessionId, + error: result.error || "Unknown error", + }); + } + + const logAction = mapActionToLogAction(result.action, result.success); + writeRecoveryLog( + recoveryConfig.logPath, + createLogEntry(assessment.sessionId, logAction, { + previousStatus: assessment.metadataStatus, + reason: result.reason ?? assessment.reason, + error: result.error, + }), + ); + } + + report.durationMs = Date.now() - startTime; + + return { + report, + assessments, + results, + recoveredSessions, + }; +} + +function mapActionToLogAction( + action: string, + success: boolean, +): "recovered" | "cleaned_up" | "escalated" | "skipped" | "error" { + if (!success) return "error"; + switch (action) { + case "recover": + return "recovered"; + case "cleanup": + return "cleaned_up"; + case "escalate": + return "escalated"; + default: + return "skipped"; + } +} + +export async function recoverSessionById( + sessionId: string, + options: RecoveryManagerOptions, +): Promise { + const { config, registry, dryRun = false } = options; + + const recoveryConfig: RecoveryConfig = { + ...DEFAULT_RECOVERY_CONFIG, + ...options.recoveryConfig, + logPath: options.recoveryConfig?.logPath ?? getRecoveryLogPath(config.configPath), + }; + + const context: RecoveryContext = { + configPath: config.configPath, + recoveryConfig, + dryRun, + }; + + const allSessions = scanAllSessions(config); + const scanned = allSessions.find((s) => s.sessionId === sessionId); + + if (!scanned) { + return null; + } + + const assessment = await validateSession(scanned, config, registry, recoveryConfig); + + const result = await executeAction(assessment, config, registry, context); + + if (dryRun) { + return result; + } + + const logAction = mapActionToLogAction(result.action, result.success); + writeRecoveryLog( + recoveryConfig.logPath, + createLogEntry(sessionId, logAction, { + previousStatus: assessment.metadataStatus, + reason: result.reason ?? assessment.reason, + error: result.error, + }), + ); + + return result; +} diff --git a/packages/core/src/recovery/scanner.ts b/packages/core/src/recovery/scanner.ts new file mode 100644 index 000000000..79a56c56b --- /dev/null +++ b/packages/core/src/recovery/scanner.ts @@ -0,0 +1,48 @@ +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { SessionId, OrchestratorConfig, ProjectConfig } from "../types.js"; +import { listMetadata, readMetadataRaw } from "../metadata.js"; +import { getSessionsDir, generateConfigHash } from "../paths.js"; + +export interface ScannedSession { + sessionId: SessionId; + projectId: string; + project: ProjectConfig; + sessionsDir: string; + rawMetadata: Record; +} + +export function scanAllSessions( + config: OrchestratorConfig, + projectIdFilter?: string, +): ScannedSession[] { + const results: ScannedSession[] = []; + + for (const [projectKey, project] of Object.entries(config.projects)) { + if (projectIdFilter && projectKey !== projectIdFilter) continue; + + const sessionsDir = getSessionsDir(config.configPath, project.path); + if (!existsSync(sessionsDir)) continue; + + for (const file of listMetadata(sessionsDir)) { + const rawMetadata = readMetadataRaw(sessionsDir, file); + if (!rawMetadata) continue; + + results.push({ + sessionId: file, + projectId: projectKey, + project, + sessionsDir, + rawMetadata, + }); + } + } + + return results; +} + +export function getRecoveryLogPath(configPath: string): string { + const hash = generateConfigHash(configPath); + return join(homedir(), ".agent-orchestrator", hash, "recovery.log"); +} diff --git a/packages/core/src/recovery/types.ts b/packages/core/src/recovery/types.ts new file mode 100644 index 000000000..8518a9940 --- /dev/null +++ b/packages/core/src/recovery/types.ts @@ -0,0 +1,216 @@ +/** + * Recovery Types — Session classification and recovery result types. + * + * Part of the orchestrator recovery automation system (Issue #356). + * These types define how sessions are classified and what actions are taken. + */ + +import type { SessionId, SessionStatus, Session, RuntimeHandle, ActivityState } from "../types.js"; + +/** + * Classification of a session's recoverability state. + * + * - live: Session is running normally, just needs to be re-registered + * - dead: Runtime is gone, needs cleanup + * - partial: Some components exist but session is incomplete + * - unrecoverable: Session is in a terminal state (merged, done) - skip + */ +export type RecoveryClassification = "live" | "dead" | "partial" | "unrecoverable"; + +/** + * Action to take for a session during recovery. + * + * - recover: Restore session to in-memory state + * - cleanup: Remove runtime/workspace, archive metadata + * - escalate: Requires manual intervention + * - skip: No action needed + */ +export type RecoveryAction = "recover" | "cleanup" | "escalate" | "skip"; + +/** + * Assessment of a session's state for recovery purposes. + */ +export interface RecoveryAssessment { + /** Session ID being assessed */ + sessionId: SessionId; + + /** Project ID this session belongs to */ + projectId: string; + + /** Overall classification */ + classification: RecoveryClassification; + + /** Recommended action */ + action: RecoveryAction; + + /** Human-readable reason for classification */ + reason: string; + + // --- Runtime state --- + + /** Whether the runtime (tmux/docker) is alive */ + runtimeAlive: boolean; + + /** Runtime handle if available */ + runtimeHandle: RuntimeHandle | null; + + // --- Workspace state --- + + /** Whether the workspace directory exists */ + workspaceExists: boolean; + + /** Workspace path if known */ + workspacePath: string | null; + + // --- Agent state --- + + /** Whether the agent process appears to be running */ + agentProcessRunning: boolean; + + /** Detected agent activity state */ + agentActivity: ActivityState | null; + + // --- Metadata state --- + + /** Whether metadata file is valid/readable */ + metadataValid: boolean; + + /** Current status from metadata */ + metadataStatus: SessionStatus; + + /** Raw metadata key-value pairs */ + rawMetadata: Record; +} + +/** + * Result of attempting to recover a single session. + */ +export interface RecoveryResult { + /** Whether the operation succeeded */ + success: boolean; + + /** Session ID that was processed */ + sessionId: SessionId; + + /** Action that was taken */ + action: RecoveryAction; + + /** Recovered session object (only for 'recover' action) */ + session?: Session; + + /** Whether manual intervention is required */ + requiresManualIntervention?: boolean; + + reason?: string; + + /** Error message if operation failed */ + error?: string; +} + +/** + * Summary report of a recovery operation. + */ +export interface RecoveryReport { + /** When the recovery was run */ + timestamp: Date; + + /** Total sessions scanned */ + totalScanned: number; + + /** Sessions that were recovered (live, restored to memory) */ + recovered: SessionId[]; + + /** Sessions that were cleaned up (dead, resources removed) */ + cleanedUp: SessionId[]; + + /** Sessions that require manual intervention */ + escalated: SessionId[]; + + /** Sessions that were skipped (no action needed) */ + skipped: SessionId[]; + + /** Errors encountered during recovery */ + errors: Array<{ sessionId: SessionId; error: string }>; + + /** Time taken for recovery in milliseconds */ + durationMs: number; +} + +/** + * Entry in the recovery log. + */ +export interface RecoveryLogEntry { + /** ISO timestamp */ + timestamp: string; + + /** Session ID */ + sessionId: SessionId; + + /** Action taken */ + action: "recovered" | "cleaned_up" | "escalated" | "skipped" | "error"; + + /** Previous status (for recovered sessions) */ + previousStatus?: SessionStatus; + + /** Reason for action (for cleanup/escalate) */ + reason?: string; + + /** Error message (for error action) */ + error?: string; + + /** Additional details */ + details?: Record; +} + +/** + * Configuration for recovery behavior. + */ +export interface RecoveryConfig { + /** Enable automatic recovery on orchestrator startup */ + enabled: boolean; + + /** Maximum time for recovery phase in milliseconds */ + timeoutMs: number; + + /** Number of concurrent validation tasks */ + parallelValidation: number; + + /** Path to recovery log file */ + logPath: string; + + /** Automatically cleanup dead sessions */ + autoCleanup: boolean; + + /** Escalate partial sessions (vs. auto-cleanup) */ + escalatePartial: boolean; + + /** Maximum recovery attempts before escalating */ + maxRecoveryAttempts: number; +} + +/** + * Default recovery configuration. + */ +export const DEFAULT_RECOVERY_CONFIG: RecoveryConfig = { + enabled: true, + timeoutMs: 30_000, + parallelValidation: 5, + logPath: "", // Will be set to ~/.agent-orchestrator/recovery.log + autoCleanup: true, + escalatePartial: true, + maxRecoveryAttempts: 3, +}; + +/** + * Context passed to recovery functions. + */ +export interface RecoveryContext { + /** Root config path for the orchestrator */ + configPath: string; + + /** Recovery configuration */ + recoveryConfig: RecoveryConfig; + + /** Whether this is a dry run (no actual changes) */ + dryRun: boolean; +} diff --git a/packages/core/src/recovery/validator.ts b/packages/core/src/recovery/validator.ts new file mode 100644 index 000000000..8b4534991 --- /dev/null +++ b/packages/core/src/recovery/validator.ts @@ -0,0 +1,181 @@ +import { existsSync } from "node:fs"; +import { + TERMINAL_STATUSES as TERMINAL_STATUSES_SET, + type OrchestratorConfig, + type PluginRegistry, + type Runtime, + type Agent, + type Workspace, + type RuntimeHandle, + type SessionStatus, + type ActivityState, +} from "../types.js"; +import { safeJsonParse, validateStatus } from "../utils/validation.js"; +import type { ScannedSession } from "./scanner.js"; +import { + DEFAULT_RECOVERY_CONFIG, + type RecoveryAssessment, + type RecoveryClassification, + type RecoveryAction, + type RecoveryConfig, +} from "./types.js"; + +export async function validateSession( + scanned: ScannedSession, + config: OrchestratorConfig, + registry: PluginRegistry, + recoveryConfigInput?: Partial, +): Promise { + const { sessionId, projectId, project, rawMetadata } = scanned; + + const runtimeName = project.runtime ?? config.defaults.runtime; + const agentName = rawMetadata["agent"] ?? project.agent ?? config.defaults.agent; + const workspaceName = project.workspace ?? config.defaults.workspace; + + const runtime = registry.get("runtime", runtimeName); + const agent = registry.get("agent", agentName); + const workspace = registry.get("workspace", workspaceName); + + const workspacePath = rawMetadata["worktree"] || null; + const runtimeHandleStr = rawMetadata["runtimeHandle"]; + const runtimeHandle = runtimeHandleStr ? safeJsonParse(runtimeHandleStr) : null; + const metadataStatus = validateStatus(rawMetadata["status"]); + const recoveryConfig: RecoveryConfig = { + ...DEFAULT_RECOVERY_CONFIG, + ...(recoveryConfigInput ?? {}), + }; + + let runtimeAlive = false; + if (runtime && runtimeHandle) { + try { + runtimeAlive = await runtime.isAlive(runtimeHandle); + } catch { + runtimeAlive = false; + } + } + + let workspaceExists = false; + if (workspacePath) { + try { + workspaceExists = existsSync(workspacePath); + } catch { + workspaceExists = false; + } + if (!workspaceExists && workspace?.exists) { + try { + workspaceExists = await workspace.exists(workspacePath); + } catch { + workspaceExists = false; + } + } + } + + let agentProcessRunning = false; + const agentActivity: ActivityState | null = null; + if (agent && runtimeHandle) { + try { + agentProcessRunning = await agent.isProcessRunning(runtimeHandle); + } catch { + agentProcessRunning = false; + } + } + + const metadataValid = Object.keys(rawMetadata).length > 0; + const classification = classifySession( + runtimeAlive, + workspaceExists, + agentProcessRunning, + metadataStatus, + ); + const action = determineAction(classification, metadataStatus, recoveryConfig); + + return { + sessionId, + projectId, + classification, + action, + reason: getReason(classification, runtimeAlive, workspaceExists, agentProcessRunning), + runtimeAlive, + runtimeHandle, + workspaceExists, + workspacePath, + agentProcessRunning, + agentActivity, + metadataValid, + metadataStatus, + rawMetadata, + }; +} + +function classifySession( + runtimeAlive: boolean, + workspaceExists: boolean, + agentProcessRunning: boolean, + metadataStatus: SessionStatus, +): RecoveryClassification { + if (runtimeAlive && workspaceExists && agentProcessRunning) { + return "live"; + } + + if (!runtimeAlive && !workspaceExists) { + if (TERMINAL_STATUSES_SET.has(metadataStatus)) { + return "unrecoverable"; + } + return "dead"; + } + + if (runtimeAlive && !workspaceExists) { + return "partial"; + } + + if (!runtimeAlive && workspaceExists) { + return "dead"; + } + + if (runtimeAlive && workspaceExists && !agentProcessRunning) { + return "partial"; + } + + return "partial"; +} + +function determineAction( + classification: RecoveryClassification, + _metadataStatus: SessionStatus, + recoveryConfig: RecoveryConfig, +): RecoveryAction { + switch (classification) { + case "live": + return "recover"; + case "dead": + return recoveryConfig.autoCleanup ? "cleanup" : "escalate"; + case "partial": + return recoveryConfig.escalatePartial ? "escalate" : "cleanup"; + case "unrecoverable": + return "skip"; + default: + return "skip"; + } +} + +function getReason( + classification: RecoveryClassification, + runtimeAlive: boolean, + workspaceExists: boolean, + agentProcessRunning: boolean, +): string { + switch (classification) { + case "live": + return "Session is running normally"; + case "dead": + return `Runtime ${runtimeAlive ? "alive" : "dead"}, workspace ${workspaceExists ? "exists" : "missing"}`; + case "partial": + return `Incomplete state: runtime=${runtimeAlive}, workspace=${workspaceExists}, agent=${agentProcessRunning}`; + case "unrecoverable": + return "Session is in terminal state"; + default: + return "Unknown classification"; + } +} + +export { classifySession, determineAction }; diff --git a/packages/core/src/session-manager.ts b/packages/core/src/session-manager.ts index 3a072f9cb..4b5242330 100644 --- a/packages/core/src/session-manager.ts +++ b/packages/core/src/session-manager.ts @@ -28,7 +28,6 @@ import { type SessionId, type SessionSpawnConfig, type OrchestratorSpawnConfig, - type SessionStatus, type CleanupResult, type ClaimPROptions, type ClaimPRResult, @@ -71,6 +70,8 @@ import { GLOBAL_PAUSE_SOURCE_KEY, parsePauseUntil, } from "./global-pause.js"; +import { sessionFromMetadata } from "./utils/session-from-metadata.js"; +import { safeJsonParse } from "./utils/validation.js"; const execFileAsync = promisify(execFile); const OPENCODE_DISCOVERY_TIMEOUT_MS = 2_000; @@ -191,35 +192,6 @@ function getNextSessionNumber(existingSessions: string[], prefix: string): numbe return max + 1; } -/** Safely parse JSON, returning null on failure. */ -function safeJsonParse(str: string): T | null { - try { - return JSON.parse(str) as T; - } catch { - return null; - } -} - -/** Valid session statuses for validation. */ -const VALID_STATUSES: ReadonlySet = new Set([ - "spawning", - "working", - "pr_open", - "ci_failed", - "review_pending", - "changes_requested", - "approved", - "mergeable", - "merged", - "cleanup", - "needs_input", - "stuck", - "errored", - "killed", - "done", - "terminated", -]); - const PR_TRACKING_STATUSES: ReadonlySet = new Set([ "pr_open", "ci_failed", @@ -239,14 +211,6 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -/** Validate and normalize a status string. */ -function validateStatus(raw: string | undefined): SessionStatus { - // Bash scripts write "starting" — treat as "working" - if (raw === "starting") return "working"; - if (raw && VALID_STATUSES.has(raw)) return raw as SessionStatus; - return "spawning"; -} - /** Reconstruct a Session object from raw metadata key=value pairs. */ function metadataToSession( sessionId: SessionId, @@ -254,42 +218,10 @@ function metadataToSession( createdAt?: Date, modifiedAt?: Date, ): Session { - return { - id: sessionId, - projectId: meta["project"] ?? "", - status: validateStatus(meta["status"]), - activity: null, - branch: meta["branch"] || null, - issueId: meta["issue"] || null, - pr: meta["pr"] - ? (() => { - // Parse owner/repo from GitHub PR URL: https://github.com/owner/repo/pull/123 - const prUrl = meta["pr"]; - const ghMatch = prUrl.match(/github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/); - return { - number: ghMatch - ? parseInt(ghMatch[3], 10) - : parseInt(prUrl.match(/\/(\d+)$/)?.[1] ?? "0", 10), - url: prUrl, - title: "", - owner: ghMatch?.[1] ?? "", - repo: ghMatch?.[2] ?? "", - branch: meta["branch"] ?? "", - baseBranch: "", - isDraft: false, - }; - })() - : null, - workspacePath: meta["worktree"] || null, - runtimeHandle: meta["runtimeHandle"] - ? safeJsonParse(meta["runtimeHandle"]) - : null, - agentInfo: meta["summary"] ? { summary: meta["summary"], agentSessionId: null } : null, - createdAt: meta["createdAt"] ? new Date(meta["createdAt"]) : (createdAt ?? new Date()), + return sessionFromMetadata(sessionId, meta, { + createdAt, lastActivityAt: modifiedAt ?? new Date(), - restoredAt: meta["restoredAt"] ? new Date(meta["restoredAt"]) : undefined, - metadata: meta, - }; + }); } export interface SessionManagerDeps { diff --git a/packages/core/src/utils/pr.ts b/packages/core/src/utils/pr.ts new file mode 100644 index 000000000..0f4fd977d --- /dev/null +++ b/packages/core/src/utils/pr.ts @@ -0,0 +1,31 @@ +import type { PRInfo } from "../types.js"; + +export type ParsedPrUrl = Pick; + +const GITHUB_PR_URL_REGEX = /github\.com\/([^/]+)\/([^/]+)\/pull\/(\d+)/; +const TRAILING_NUMBER_REGEX = /\/(\d+)$/; + +export function parsePrFromUrl(prUrl: string): ParsedPrUrl | null { + const githubMatch = prUrl.match(GITHUB_PR_URL_REGEX); + if (githubMatch) { + const [, owner, repo, prNumber] = githubMatch; + return { + owner, + repo, + number: parseInt(prNumber, 10), + url: prUrl, + }; + } + + const trailingNumberMatch = prUrl.match(TRAILING_NUMBER_REGEX); + if (trailingNumberMatch) { + return { + owner: "", + repo: "", + number: parseInt(trailingNumberMatch[1], 10), + url: prUrl, + }; + } + + return null; +} diff --git a/packages/core/src/utils/session-from-metadata.ts b/packages/core/src/utils/session-from-metadata.ts new file mode 100644 index 000000000..50996fda8 --- /dev/null +++ b/packages/core/src/utils/session-from-metadata.ts @@ -0,0 +1,55 @@ +import type { RuntimeHandle, Session, SessionId, SessionStatus } from "../types.js"; +import { parsePrFromUrl } from "./pr.js"; +import { safeJsonParse, validateStatus } from "./validation.js"; + +interface SessionFromMetadataOptions { + status?: SessionStatus; + activity?: Session["activity"]; + runtimeHandle?: RuntimeHandle | null; + createdAt?: Date; + lastActivityAt?: Date; + restoredAt?: Date; +} + +export function sessionFromMetadata( + sessionId: SessionId, + meta: Record, + options: SessionFromMetadataOptions = {}, +): Session { + return { + id: sessionId, + projectId: meta["project"] ?? "", + status: options.status ?? validateStatus(meta["status"]), + activity: options.activity ?? null, + branch: meta["branch"] || null, + issueId: meta["issue"] || null, + pr: meta["pr"] + ? (() => { + const parsed = parsePrFromUrl(meta["pr"]); + return { + number: parsed?.number ?? 0, + url: meta["pr"], + title: "", + owner: parsed?.owner ?? "", + repo: parsed?.repo ?? "", + branch: meta["branch"] ?? "", + baseBranch: "", + isDraft: false, + }; + })() + : null, + workspacePath: meta["worktree"] || null, + runtimeHandle: + options.runtimeHandle !== undefined + ? options.runtimeHandle + : meta["runtimeHandle"] + ? safeJsonParse(meta["runtimeHandle"]) + : null, + agentInfo: meta["summary"] ? { summary: meta["summary"], agentSessionId: null } : null, + createdAt: meta["createdAt"] ? new Date(meta["createdAt"]) : (options.createdAt ?? new Date()), + lastActivityAt: options.lastActivityAt ?? new Date(), + restoredAt: + options.restoredAt ?? (meta["restoredAt"] ? new Date(meta["restoredAt"]) : undefined), + metadata: meta, + }; +} diff --git a/packages/core/src/utils/validation.ts b/packages/core/src/utils/validation.ts new file mode 100644 index 000000000..0e9dd744f --- /dev/null +++ b/packages/core/src/utils/validation.ts @@ -0,0 +1,42 @@ +/** + * Shared validation utilities. + */ + +import type { SessionStatus } from "../types.js"; + +/** Valid session statuses for validation. */ +const VALID_STATUSES: ReadonlySet = new Set([ + "spawning", + "working", + "pr_open", + "ci_failed", + "review_pending", + "changes_requested", + "approved", + "mergeable", + "merged", + "cleanup", + "needs_input", + "stuck", + "errored", + "killed", + "done", + "terminated", +]); + +/** Safely parse JSON, returning null on failure. */ +export function safeJsonParse(str: string): T | null { + try { + return JSON.parse(str) as T; + } catch { + return null; + } +} + +/** Validate and normalize a status string. */ +export function validateStatus(raw: string | undefined): SessionStatus { + // Bash scripts write "starting" — treat as "working" + if (raw === "starting") return "working"; + if (raw && VALID_STATUSES.has(raw)) return raw as SessionStatus; + return "spawning"; +}